Package io.debezium.pipeline.source.snapshot.incremental

Class AbstractIncrementalSnapshotTest<T extends org.apache.kafka.connect.source.SourceConnector>

java.lang.Object

io.debezium.embedded.AbstractConnectorTest

io.debezium.pipeline.source.snapshot.incremental.AbstractIncrementalSnapshotTest<T>

All Implemented Interfaces:

Testing

Direct Known Subclasses:

AbstractIncrementalSnapshotWithSchemaChangesSupportTest

public abstract class AbstractIncrementalSnapshotTest<T extends org.apache.kafka.connect.source.SourceConnector>
extends AbstractConnectorTest

Nested Class Summary

Nested classes/interfaces inherited from class io.debezium.embedded.AbstractConnectorTest

AbstractConnectorTest.SourceRecords

Nested classes/interfaces inherited from interface io.debezium.util.Testing

Testing.Debug, Testing.Files, Testing.InterruptableFunction, Testing.Network, Testing.Print, Testing.Timer

Field Summary

Fields

Modifier and Type	Field	Description
protected static KafkaCluster	kafka
private static final int	MAXIMUM_NO_RECORDS_CONSUMES
private static final int	PARTITION_NO
protected static final int	ROW_COUNT
protected static final Path	SCHEMA_HISTORY_PATH
private static final String	SERVER_NAME

Fields inherited from class io.debezium.embedded.AbstractConnectorTest

consumedLines, engine, logger, logTestName, OFFSET_STORE_PATH, pollTimeoutInMs, skipTestRule

Constructor Summary

Constructors

Constructor	Description
AbstractIncrementalSnapshotTest()

Method Summary

Modifier and Type	Method	Description
protected String	alterTableAddColumnStatement(String tableName)
protected String	alterTableDropColumnStatement(String tableName)
private void	assertCorrectIncrementalSnapshotNotification(List<org.apache.kafka.connect.source.SourceRecord> notifications)
protected abstract Configuration.Builder	config()
protected abstract Class<T>	connectorClass()
protected boolean	consumeAnyRemainingIncrementalSnapshotEventsAndCheckForStopMessage(LogInterceptor interceptor, String stopMessage)
protected Map<Integer,Integer>	consumeMixedWithIncrementalSnapshot(int recordCount)
protected Map<Integer,Integer>	consumeMixedWithIncrementalSnapshot(int recordCount, String topicName)
protected <V> Map<Integer,V>	consumeMixedWithIncrementalSnapshot(int recordCount, Function<org.apache.kafka.connect.source.SourceRecord,V> valueConverter, Predicate<Map.Entry<Integer,V>> dataCompleted, Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer, String topicName)
protected Map<Integer,Integer>	consumeMixedWithIncrementalSnapshot(int recordCount, Predicate<Map.Entry<Integer,Integer>> dataCompleted, Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
protected <V> Map<Integer,V>	consumeMixedWithIncrementalSnapshot(int recordCount, Predicate<Map.Entry<Integer,V>> dataCompleted, Function<org.apache.kafka.connect.data.Struct,Integer> idCalculator, Function<org.apache.kafka.connect.source.SourceRecord,V> valueConverter, String topicName, Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
protected Map<Integer,org.apache.kafka.connect.source.SourceRecord>	consumeRecordsMixedWithIncrementalSnapshot(int recordCount)
protected Map<Integer,org.apache.kafka.connect.source.SourceRecord>	consumeRecordsMixedWithIncrementalSnapshot(int recordCount, Predicate<Map.Entry<Integer,org.apache.kafka.connect.source.SourceRecord>> dataCompleted, Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
protected abstract JdbcConnection	databaseConnection()
protected int	defaultIncrementalSnapshotChunkSize()
protected int	getMaximumEnqueuedRecordCount()	Get the maximum number of messages that can be obtained from the connector and held in-memory before they are consumed by test methods using AbstractConnectorTest.consumeRecord(), AbstractConnectorTest.consumeRecords(int), or AbstractConnectorTest.consumeRecords(int, Consumer).
private Function<org.apache.kafka.connect.data.Struct,Integer>	getRecordValue()
protected String	getSignalsTopic()
protected String	getSignalTypeFieldName()
private static BiPredicate<Integer,org.apache.kafka.connect.source.SourceRecord>	incrementalSnapshotCompleted()
void	inserts()
void	insertsWithKafkaSnapshotSignal()
void	invalidTablesInTheList()
protected abstract Configuration.Builder	mutableConfig(boolean signalTableOnly, boolean storeOnlyCapturedDdl)
protected String	pkFieldName()
protected void	populate4PkTable(JdbcConnection connection, String tableName)
protected void	populateTable()
protected void	populateTable(JdbcConnection connection)
protected void	populateTable(JdbcConnection connection, String tableName)
protected void	populateTables()
protected void	populateTables(JdbcConnection connection)
protected void	populateTableWithSpecificValue(int startRow, int count, int value)
private void	populateTableWithSpecificValue(JdbcConnection connection, String tableName, int startRow, int count, int value)
void	removeNotYetCapturedCollectionFromInProgressIncrementalSnapshot()
void	removeStartedCapturedCollectionFromInProgressIncrementalSnapshot()
protected void	sendAdHocKafkaSnapshotSignal()
protected void	sendAdHocSnapshotSignal()
protected void	sendAdHocSnapshotSignal(String... dataCollectionIds)
protected void	sendAdHocSnapshotSignalAndWait(String... collectionIds)
protected void	sendAdHocSnapshotSignalWithAdditionalConditionWithSurrogateKey(Optional<String> additionalCondition, Optional<String> surrogateKey, String... dataCollectionIds)
protected void	sendAdHocSnapshotStopSignal(String... dataCollectionIds)
protected void	sendAdHocSnapshotStopSignalAndWait(String... collectionIds)
protected void	sendExecuteSnapshotKafkaSignal(String fullTableNames)
protected void	sendKafkaSignal(String signalValue)
protected void	sendPauseSignal()
protected void	sendResumeSignal()
void	shouldExecuteRegularSnapshotWhenAdditionalConditionEmpty()
void	shouldSnapshotNewlyAddedTableToIncludeListAfterRestart()
protected abstract String	signalTableName()
protected String	signalTableNameSanitized()
void	snapshotOnly()
void	snapshotOnlyWithRestart()
void	snapshotPreceededBySchemaChange()
void	snapshotWithAdditionalCondition()
void	snapshotWithAdditionalConditionWithRestart()
void	snapshotWithAdditionalConditionWithSurrogateKey()
void	snapshotWithRegexDataCollections()
void	snapshotWithSurrogateKey()
protected void	startConnector()
protected void	startConnector(io.debezium.engine.DebeziumEngine.CompletionCallback callback)
protected void	startConnector(Function<Configuration.Builder,Configuration.Builder> custConfig)
protected void	startConnector(Function<Configuration.Builder,Configuration.Builder> custConfig, io.debezium.engine.DebeziumEngine.CompletionCallback callback, boolean expectNoRecords)
protected void	startConnectorWithSnapshot(Function<Configuration.Builder,Configuration.Builder> custConfig)
void	stopCurrentIncrementalSnapshotWithAllCollectionsAndTakeNewNewIncrementalSnapshotAfterRestart()
void	stopCurrentIncrementalSnapshotWithoutCollectionsAndTakeNewNewIncrementalSnapshotAfterRestart()
protected String	tableDataCollectionId()
protected List<String>	tableDataCollectionIds()
protected abstract String	tableName()
protected abstract List<String>	tableNames()
void	testNotification()
void	testPauseDuringSnapshot()
protected abstract String	topicName()
protected abstract List<String>	topicNames()
void	updates()
void	updatesLargeChunk()
void	updatesWithRestart()
protected String	valueFieldName()
protected void	waitForCdcTransactionPropagation(int expectedTransactions)
protected void	waitForConnectorToStart()

Methods inherited from class io.debezium.embedded.AbstractConnectorTest

assertBeginTransaction, assertConfigurationErrors, assertConfigurationErrors, assertConfigurationErrors, assertConnectorIsRunning, assertConnectorNotRunning, assertDelete, assertEndTransaction, assertEngineIsRunning, assertHasNoSourceQuery, assertInsert, assertKey, assertNoConfigurationErrors, assertNoRecordsToConsume, assertOffset, assertOffset, assertOnlyTransactionRecordsToConsume, assertRecordTransactionMetadata, assertSchemaMatchesStruct, assertSchemaMatchesStruct, assertSourceQuery, assertTombstone, assertTombstone, assertUpdate, assertValueField, configValue, consumeAvailableRecords, consumeDmlRecordsByTopic, consumeDmlRecordsByTopic, consumeDmlRecordsByTopic, consumeRecord, consumeRecords, consumeRecords, consumeRecords, consumeRecordsButSkipUntil, consumeRecordsByTopic, consumeRecordsByTopic, consumeRecordsByTopic, consumeRecordsByTopicUntil, consumeRecordsUntil, debug, getConsumer, getSnapshotMetricsObjectName, getSnapshotMetricsObjectName, getStreamingMetricsObjectName, getStreamingMetricsObjectName, getStreamingNamespace, initializeConnectorTestFramework, isStreamingRunning, isStreamingRunning, isTransactionRecord, loggingCompletion, print, readLastCommittedOffset, readLastCommittedOffsets, setConsumeTimeout, skipAvroValidation, start, start, start, start, start, start, start, startAndConsumeTillEnd, startAndConsumeTillEnd, stopConnector, stopConnector, validate, waitForAvailableRecords, waitForConnectorShutdown, waitForNotInitialState, waitForSnapshotToBeCompleted, waitForSnapshotToBeCompleted, waitForStreamingRunning, waitForStreamingRunning, waitTimeForRecords, waitTimeForRecordsAfterNulls

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface io.debezium.util.Testing

once, once, resetBeforeEachTest, time, time

Field Details

ROW_COUNT

protected static final int ROW_COUNT

See Also:

• Constant Field Values

MAXIMUM_NO_RECORDS_CONSUMES

private static final int MAXIMUM_NO_RECORDS_CONSUMES

See Also:

• Constant Field Values

SCHEMA_HISTORY_PATH

protected static final Path SCHEMA_HISTORY_PATH

PARTITION_NO

private static final int PARTITION_NO

See Also:

• Constant Field Values

SERVER_NAME

private static final String SERVER_NAME

See Also:

• Constant Field Values

kafka

protected static KafkaCluster kafka

Constructor Details

AbstractIncrementalSnapshotTest

public AbstractIncrementalSnapshotTest()

Method Details

connectorClass

protected abstract Class<T> connectorClass()

databaseConnection

protected abstract JdbcConnection databaseConnection()

topicName

protected abstract String topicName()

tableName

protected abstract String tableName()

topicNames

protected abstract List<String> topicNames()

tableNames

protected abstract List<String> tableNames()

signalTableName

protected abstract String signalTableName()

signalTableNameSanitized

protected String signalTableNameSanitized()

config

protected abstract Configuration.Builder config()

mutableConfig

protected abstract Configuration.Builder mutableConfig(boolean signalTableOnly,
 boolean storeOnlyCapturedDdl)

waitForCdcTransactionPropagation

protected void waitForCdcTransactionPropagation(int expectedTransactions)
                                         throws Exception

Throws:

Exception

alterTableAddColumnStatement

protected String alterTableAddColumnStatement(String tableName)

alterTableDropColumnStatement

protected String alterTableDropColumnStatement(String tableName)

tableDataCollectionId

protected String tableDataCollectionId()

tableDataCollectionIds

protected List<String> tableDataCollectionIds()

populateTable

protected void populateTable(JdbcConnection connection,
 String tableName)
                      throws SQLException

Throws:

SQLException

populateTable

protected void populateTable(JdbcConnection connection)
                      throws SQLException

Throws:

SQLException

populateTables

protected void populateTables(JdbcConnection connection)
                       throws SQLException

Throws:

SQLException

populateTable

protected void populateTable()
                      throws SQLException

Throws:

SQLException

populateTableWithSpecificValue

protected void populateTableWithSpecificValue(int startRow,
 int count,
 int value)
                                       throws SQLException

Throws:

SQLException

populateTableWithSpecificValue

private void populateTableWithSpecificValue(JdbcConnection connection,
 String tableName,
 int startRow,
 int count,
 int value)
                                     throws SQLException

Throws:

SQLException

populateTables

protected void populateTables()
                       throws SQLException

Throws:

SQLException

populate4PkTable

protected void populate4PkTable(JdbcConnection connection,
 String tableName)
                         throws SQLException

Throws:

SQLException

consumeMixedWithIncrementalSnapshot

protected Map<Integer,Integer> consumeMixedWithIncrementalSnapshot(int recordCount)
                                                            throws InterruptedException

Throws:

InterruptedException

consumeMixedWithIncrementalSnapshot

protected Map<Integer,Integer> consumeMixedWithIncrementalSnapshot(int recordCount,
 String topicName)
                                                            throws InterruptedException

Throws:

InterruptedException

consumeMixedWithIncrementalSnapshot

protected <V> Map<Integer,V> consumeMixedWithIncrementalSnapshot(int recordCount,
 Function<org.apache.kafka.connect.source.SourceRecord,V> valueConverter,
 Predicate<Map.Entry<Integer,V>> dataCompleted,
 Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer,
 String topicName)
                                                          throws InterruptedException

Throws:

InterruptedException

consumeMixedWithIncrementalSnapshot

protected <V> Map<Integer,V> consumeMixedWithIncrementalSnapshot(int recordCount,
 Predicate<Map.Entry<Integer,V>> dataCompleted,
 Function<org.apache.kafka.connect.data.Struct,Integer> idCalculator,
 Function<org.apache.kafka.connect.source.SourceRecord,V> valueConverter,
 String topicName,
 Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
                                                          throws InterruptedException

Throws:

InterruptedException

consumeRecordsMixedWithIncrementalSnapshot

protected Map<Integer,org.apache.kafka.connect.source.SourceRecord> consumeRecordsMixedWithIncrementalSnapshot(int recordCount)
                                                                                                        throws InterruptedException

Throws:

InterruptedException

consumeMixedWithIncrementalSnapshot

protected Map<Integer,Integer> consumeMixedWithIncrementalSnapshot(int recordCount,
 Predicate<Map.Entry<Integer,Integer>> dataCompleted,
 Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
                                                            throws InterruptedException

Throws:

InterruptedException

consumeRecordsMixedWithIncrementalSnapshot

protected Map<Integer,org.apache.kafka.connect.source.SourceRecord> consumeRecordsMixedWithIncrementalSnapshot(int recordCount,
 Predicate<Map.Entry<Integer,org.apache.kafka.connect.source.SourceRecord>> dataCompleted,
 Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
                                                                                                        throws InterruptedException

Throws:

InterruptedException

valueFieldName

protected String valueFieldName()

pkFieldName

protected String pkFieldName()

getSignalTypeFieldName

protected String getSignalTypeFieldName()

sendAdHocSnapshotSignal

protected void sendAdHocSnapshotSignal(String... dataCollectionIds)
                                throws SQLException

Throws:

SQLException

sendAdHocSnapshotSignalWithAdditionalConditionWithSurrogateKey

protected void sendAdHocSnapshotSignalWithAdditionalConditionWithSurrogateKey(Optional<String> additionalCondition,
 Optional<String> surrogateKey,
 String... dataCollectionIds)

sendAdHocSnapshotStopSignal

protected void sendAdHocSnapshotStopSignal(String... dataCollectionIds)
                                    throws SQLException

Throws:

SQLException

sendAdHocSnapshotSignal

protected void sendAdHocSnapshotSignal()
                                throws SQLException

Throws:

SQLException

sendAdHocKafkaSnapshotSignal

protected void sendAdHocKafkaSnapshotSignal()
                                     throws ExecutionException,
InterruptedException

Throws:

ExecutionException

InterruptedException

sendExecuteSnapshotKafkaSignal

protected void sendExecuteSnapshotKafkaSignal(String fullTableNames)
                                       throws ExecutionException,
InterruptedException

Throws:

ExecutionException

InterruptedException

getSignalsTopic

protected String getSignalsTopic()

sendKafkaSignal

protected void sendKafkaSignal(String signalValue)
                        throws ExecutionException,
InterruptedException

Throws:

ExecutionException

InterruptedException

sendPauseSignal

protected void sendPauseSignal()

sendResumeSignal

protected void sendResumeSignal()

startConnector

protected void startConnector(io.debezium.engine.DebeziumEngine.CompletionCallback callback)

startConnector

protected void startConnector(Function<Configuration.Builder,Configuration.Builder> custConfig)

startConnector

protected void startConnector(Function<Configuration.Builder,Configuration.Builder> custConfig,
 io.debezium.engine.DebeziumEngine.CompletionCallback callback,
 boolean expectNoRecords)

startConnectorWithSnapshot

protected void startConnectorWithSnapshot(Function<Configuration.Builder,Configuration.Builder> custConfig)

startConnector

protected void startConnector()

waitForConnectorToStart

protected void waitForConnectorToStart()

snapshotOnly

public void snapshotOnly()
                  throws Exception

Throws:

Exception

invalidTablesInTheList

public void invalidTablesInTheList()
                            throws Exception

Throws:

Exception

inserts

public void inserts()
             throws Exception

Throws:

Exception

insertsWithKafkaSnapshotSignal

public void insertsWithKafkaSnapshotSignal()
                                    throws Exception

Throws:

Exception

updates

public void updates()
             throws Exception

Throws:

Exception

updatesWithRestart

public void updatesWithRestart()
                        throws Exception

Throws:

Exception

updatesLargeChunk

public void updatesLargeChunk()
                       throws Exception

Throws:

Exception

snapshotOnlyWithRestart

public void snapshotOnlyWithRestart()
                             throws Exception

Throws:

Exception

snapshotPreceededBySchemaChange

@FixFor("DBZ-4272")
public void snapshotPreceededBySchemaChange()
                                     throws Exception

Throws:

Exception

snapshotWithRegexDataCollections

public void snapshotWithRegexDataCollections()
                                      throws Exception

Throws:

Exception

stopCurrentIncrementalSnapshotWithoutCollectionsAndTakeNewNewIncrementalSnapshotAfterRestart

@FixFor("DBZ-4271")
public void stopCurrentIncrementalSnapshotWithoutCollectionsAndTakeNewNewIncrementalSnapshotAfterRestart()
                                                                                                  throws Exception

Throws:

Exception

stopCurrentIncrementalSnapshotWithAllCollectionsAndTakeNewNewIncrementalSnapshotAfterRestart

@FixFor("DBZ-4271")
public void stopCurrentIncrementalSnapshotWithAllCollectionsAndTakeNewNewIncrementalSnapshotAfterRestart()
                                                                                                  throws Exception

Throws:

Exception

removeNotYetCapturedCollectionFromInProgressIncrementalSnapshot

@FixFor("DBZ-4271")
public void removeNotYetCapturedCollectionFromInProgressIncrementalSnapshot()
                                                                     throws Exception

Throws:

Exception

removeStartedCapturedCollectionFromInProgressIncrementalSnapshot

@FixFor("DBZ-4271")
public void removeStartedCapturedCollectionFromInProgressIncrementalSnapshot()
                                                                      throws Exception

Throws:

Exception

shouldSnapshotNewlyAddedTableToIncludeListAfterRestart

@FixFor("DBZ-4834")
public void shouldSnapshotNewlyAddedTableToIncludeListAfterRestart()
                                                            throws Exception

Throws:

Exception

testPauseDuringSnapshot

public void testPauseDuringSnapshot()
                             throws Exception

Throws:

Exception

snapshotWithAdditionalCondition

public void snapshotWithAdditionalCondition()
                                     throws Exception

Throws:

Exception

shouldExecuteRegularSnapshotWhenAdditionalConditionEmpty

public void shouldExecuteRegularSnapshotWhenAdditionalConditionEmpty()
                                                              throws Exception

Throws:

Exception

snapshotWithAdditionalConditionWithRestart

public void snapshotWithAdditionalConditionWithRestart()
                                                throws Exception

Throws:

Exception

snapshotWithSurrogateKey

public void snapshotWithSurrogateKey()
                              throws Exception

Throws:

Exception

snapshotWithAdditionalConditionWithSurrogateKey

public void snapshotWithAdditionalConditionWithSurrogateKey()
                                                     throws Exception

Throws:

Exception

testNotification

public void testNotification()
                      throws Exception

Throws:

Exception

defaultIncrementalSnapshotChunkSize

protected int defaultIncrementalSnapshotChunkSize()

incrementalSnapshotCompleted

private static BiPredicate<Integer,org.apache.kafka.connect.source.SourceRecord> incrementalSnapshotCompleted()

assertCorrectIncrementalSnapshotNotification

private void assertCorrectIncrementalSnapshotNotification(List<org.apache.kafka.connect.source.SourceRecord> notifications)

getRecordValue

private Function<org.apache.kafka.connect.data.Struct,Integer> getRecordValue()

sendAdHocSnapshotSignalAndWait

protected void sendAdHocSnapshotSignalAndWait(String... collectionIds)
                                       throws Exception

Throws:

Exception

sendAdHocSnapshotStopSignalAndWait

protected void sendAdHocSnapshotStopSignalAndWait(String... collectionIds)
                                           throws Exception

Throws:

Exception

consumeAnyRemainingIncrementalSnapshotEventsAndCheckForStopMessage

protected boolean consumeAnyRemainingIncrementalSnapshotEventsAndCheckForStopMessage(LogInterceptor interceptor,
 String stopMessage)
                                                                              throws Exception

Throws:

Exception

getMaximumEnqueuedRecordCount

protected int getMaximumEnqueuedRecordCount()

Description copied from class: AbstractConnectorTest

Get the maximum number of messages that can be obtained from the connector and held in-memory before they are consumed by test methods using AbstractConnectorTest.consumeRecord(), AbstractConnectorTest.consumeRecords(int), or AbstractConnectorTest.consumeRecords(int, Consumer).

By default this method return 100.

Overrides:

getMaximumEnqueuedRecordCount in class AbstractConnectorTest

Returns:

the maximum number of records that can be enqueued