Package io.debezium.pipeline.source.snapshot.incremental

Class AbstractIncrementalSnapshotTest<T extends org.apache.kafka.connect.source.SourceConnector>

java.lang.Object

io.debezium.embedded.AbstractConnectorTest

io.debezium.embedded.async.AbstractAsyncEngineConnectorTest

io.debezium.pipeline.source.snapshot.incremental.AbstractSnapshotTest<T>

io.debezium.pipeline.source.snapshot.incremental.AbstractIncrementalSnapshotTest<T>

All Implemented Interfaces:

Testing

Direct Known Subclasses:

AbstractIncrementalSnapshotWithSchemaChangesSupportTest

public abstract class AbstractIncrementalSnapshotTest<T extends org.apache.kafka.connect.source.SourceConnector>
extends AbstractSnapshotTest<T>

Nested Class Summary

Nested classes/interfaces inherited from class io.debezium.embedded.AbstractConnectorTest

AbstractConnectorTest.SourceRecords

Nested classes/interfaces inherited from interface io.debezium.util.Testing

Testing.Debug, Testing.Files, Testing.InterruptableFunction, Testing.Network, Testing.Print, Testing.Timer

Field Summary

Fields

Modifier and Type	Field	Description
protected static KafkaCluster	kafka

Fields inherited from class io.debezium.pipeline.source.snapshot.incremental.AbstractSnapshotTest

PARTITION_NO, ROW_COUNT, SCHEMA_HISTORY_PATH, SERVER_NAME, signalsFile

Fields inherited from class io.debezium.embedded.AbstractConnectorTest

consumedLines, engine, isEngineRunning, logger, logTestName, OFFSET_STORE_PATH, pollTimeoutInMs, skipTestRule

Constructor Summary

Constructors

Constructor	Description
AbstractIncrementalSnapshotTest()

Method Summary

Modifier and Type	Method	Description
private void	assertCorrectIncrementalSnapshotNotification(List<org.apache.kafka.connect.source.SourceRecord> notifications)
private void	assertOpenCloseEventCount(JdbcConnection.ResultSetConsumer consumer)
protected boolean	consumeAnyRemainingIncrementalSnapshotEventsAndCheckForStopMessage(LogInterceptor interceptor, String stopMessage)
protected int	defaultIncrementalSnapshotChunkSize()
protected String	getSignalsTopic()
protected String	getSignalTypeFieldName()
private static BiPredicate<Integer,org.apache.kafka.connect.source.SourceRecord>	incrementalSnapshotCompleted()
void	insertDeleteWatermarkingStrategy()
void	insertInsertWatermarkingStrategy()
void	inserts()
void	insertsWithKafkaSnapshotSignal()
void	insertsWithoutPks()
void	insertsWithoutPksAndNull()
void	invalidTablesInTheList()
protected String	noPKTableDataCollectionId()
protected abstract String	noPKTableName()
protected abstract String	noPKTopicName()
void	removeNotYetCapturedCollectionFromInProgressIncrementalSnapshot()
void	removeStartedCapturedCollectionFromInProgressIncrementalSnapshot()
protected String	returnedIdentifierName(String queriedID)
protected void	sendAdHocKafkaSnapshotSignal()
protected void	sendAdHocSnapshotSignal()
protected void	sendAdHocSnapshotSignalAndWait(String... collectionIds)
protected void	sendAdHocSnapshotStopSignal(String... dataCollectionIds)
protected void	sendAdHocSnapshotStopSignalAndWait(String... collectionIds)
protected void	sendExecuteSnapshotKafkaSignal(String fullTableNames)
protected void	sendKafkaSignal(String signalValue)
protected void	sendPauseSignal()
protected void	sendResumeSignal()
void	shouldExecuteRegularSnapshotWhenAdditionalConditionEmpty()
void	shouldSnapshotNewlyAddedTableToIncludeListAfterRestart()
void	snapshotOnly()
void	snapshotOnlyWithRestart()
void	snapshotPreceededBySchemaChange()
void	snapshotWithAdditionalCondition()
void	snapshotWithAdditionalConditionWithRestart()
void	snapshotWithAdditionalConditionWithSurrogateKey()
void	snapshotWithDuplicateDataCollections()
void	snapshotWithNewAdditionalConditionsField()
void	snapshotWithRegexDataCollections()
void	snapshotWithSurrogateKey()
void	stopCurrentIncrementalSnapshotWithAllCollectionsAndTakeNewNewIncrementalSnapshotAfterRestart()
void	stopCurrentIncrementalSnapshotWithoutCollectionsAndTakeNewNewIncrementalSnapshotAfterRestart()
void	testNotification()
void	testPauseDuringSnapshot()
void	updates()
void	updatesLargeChunk()
void	updatesWithRestart()
void	whenSnapshotMultipleTablesAndConnectorRestartsThenOnlyNotAlreadyProcessedTableMustBeProcessed()

Methods inherited from class io.debezium.pipeline.source.snapshot.incremental.AbstractSnapshotTest

alterTableAddColumnStatement, alterTableDropColumnStatement, config, connector, connectorClass, consumeMixedWithIncrementalSnapshot, consumeMixedWithIncrementalSnapshot, consumeMixedWithIncrementalSnapshot, consumeMixedWithIncrementalSnapshot, consumeMixedWithIncrementalSnapshot, consumeMixedWithIncrementalSnapshot, consumeRecordsMixedWithIncrementalSnapshot, consumeRecordsMixedWithIncrementalSnapshot, database, databaseConnection, getMaximumEnqueuedRecordCount, getRecordValue, mutableConfig, pkFieldName, populate4PkTable, populateTable, populateTable, populateTable, populateTable, populateTables, populateTables, populateTableWithSpecificValue, sendAdHocSnapshotSignal, sendAdHocSnapshotSignalWithAdditionalConditionsWithSurrogateKey, sendAdHocSnapshotSignalWithAdditionalConditionsWithSurrogateKey, sendAdHocSnapshotSignalWithAdditionalConditionWithSurrogateKey, sendAdHocSnapshotSignalWithAdditionalConditionWithSurrogateKey, sendExecuteSnapshotFileSignal, sendExecuteSnapshotFileSignal, server, signalTableName, signalTableNameSanitized, startConnector, startConnector, startConnector, startConnector, startConnectorWithSnapshot, tableDataCollectionId, tableDataCollectionIds, tableName, tableNames, task, topicName, topicNames, valueFieldName, waitForCdcTransactionPropagation, waitForConnectorToStart

Methods inherited from class io.debezium.embedded.async.AbstractAsyncEngineConnectorTest

createEngine, createEngineBuilder

Methods inherited from class io.debezium.embedded.AbstractConnectorTest

assertBeginTransaction, assertConfigurationErrors, assertConfigurationErrors, assertConfigurationErrors, assertConnectorIsRunning, assertConnectorNotRunning, assertDelete, assertEndTransaction, assertEngineIsRunning, assertHasNoSourceQuery, assertInsert, assertKey, assertNoConfigurationErrors, assertNoRecordsToConsume, assertOffset, assertOffset, assertOnlyTransactionRecordsToConsume, assertRecordTransactionMetadata, assertSchemaMatchesStruct, assertSchemaMatchesStruct, assertSourceQuery, assertTombstone, assertTombstone, assertUpdate, assertValueField, configValue, consumeAvailableRecords, consumeAvailableRecordsByTopic, consumeDmlRecordsByTopic, consumeDmlRecordsByTopic, consumeDmlRecordsByTopic, consumeRecord, consumeRecords, consumeRecords, consumeRecords, consumeRecordsButSkipUntil, consumeRecordsByTopic, consumeRecordsByTopic, consumeRecordsByTopic, consumeRecordsByTopicUntil, consumeRecordsUntil, debug, getConsumer, getSnapshotMetricsObjectName, getSnapshotMetricsObjectName, getSnapshotMetricsObjectName, getStreamingMetricsObjectName, getStreamingMetricsObjectName, getStreamingMetricsObjectName, getStreamingMetricsObjectName, getStreamingNamespace, initializeConnectorTestFramework, isStreamingRunning, isStreamingRunning, isStreamingRunning, isStreamingRunning, isTransactionRecord, loggingCompletion, print, readLastCommittedOffset, readLastCommittedOffsets, setConsumeTimeout, skipAvroValidation, start, start, start, start, start, start, start, startAndConsumeTillEnd, startAndConsumeTillEnd, stopConnector, stopConnector, storeOffsets, validate, waitForAvailableRecords, waitForConnectorShutdown, waitForEngineShutdown, waitForSnapshotToBeCompleted, waitForSnapshotToBeCompleted, waitForSnapshotWithCustomMetricsToBeCompleted, waitForStreamingRunning, waitForStreamingRunning, waitForStreamingRunning, waitForStreamingWithCustomMetricsToStart, waitTimeForEngine, waitTimeForRecords, waitTimeForRecordsAfterNulls

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface io.debezium.util.Testing

once, once, resetBeforeEachTest, time, time

Field Details

kafka

protected static KafkaCluster kafka

Constructor Details

AbstractIncrementalSnapshotTest

public AbstractIncrementalSnapshotTest()

Method Details

getSignalTypeFieldName

protected String getSignalTypeFieldName()

noPKTopicName

protected abstract String noPKTopicName()

noPKTableName

protected abstract String noPKTableName()

noPKTableDataCollectionId

protected String noPKTableDataCollectionId()

returnedIdentifierName

protected String returnedIdentifierName(String queriedID)

sendAdHocSnapshotStopSignal

protected void sendAdHocSnapshotStopSignal(String... dataCollectionIds)
                                    throws SQLException

Throws:

SQLException

sendAdHocSnapshotSignal

protected void sendAdHocSnapshotSignal()
                                throws SQLException

Throws:

SQLException

sendAdHocKafkaSnapshotSignal

protected void sendAdHocKafkaSnapshotSignal()
                                     throws ExecutionException,
InterruptedException

Throws:

ExecutionException

InterruptedException

sendExecuteSnapshotKafkaSignal

protected void sendExecuteSnapshotKafkaSignal(String fullTableNames)
                                       throws ExecutionException,
InterruptedException

Throws:

ExecutionException

InterruptedException

getSignalsTopic

protected String getSignalsTopic()

sendKafkaSignal

protected void sendKafkaSignal(String signalValue)
                        throws ExecutionException,
InterruptedException

Throws:

ExecutionException

InterruptedException

sendPauseSignal

protected void sendPauseSignal()

sendResumeSignal

protected void sendResumeSignal()

snapshotOnly

public void snapshotOnly()
                  throws Exception

Throws:

Exception

invalidTablesInTheList

public void invalidTablesInTheList()
                            throws Exception

Throws:

Exception

inserts

public void inserts()
             throws Exception

Throws:

Exception

insertsWithKafkaSnapshotSignal

public void insertsWithKafkaSnapshotSignal()
                                    throws Exception

Throws:

Exception

insertsWithoutPks

public void insertsWithoutPks()
                       throws Exception

Throws:

Exception

insertsWithoutPksAndNull

public void insertsWithoutPksAndNull()
                              throws Exception

Throws:

Exception

updates

public void updates()
             throws Exception

Throws:

Exception

updatesWithRestart

public void updatesWithRestart()
                        throws Exception

Throws:

Exception

updatesLargeChunk

public void updatesLargeChunk()
                       throws Exception

Throws:

Exception

snapshotOnlyWithRestart

public void snapshotOnlyWithRestart()
                             throws Exception

Throws:

Exception

whenSnapshotMultipleTablesAndConnectorRestartsThenOnlyNotAlreadyProcessedTableMustBeProcessed

@FixFor("DBZ-7716")
public void whenSnapshotMultipleTablesAndConnectorRestartsThenOnlyNotAlreadyProcessedTableMustBeProcessed()
                                                                                                   throws Exception

Throws:

Exception

snapshotPreceededBySchemaChange

@FixFor("DBZ-4272")
public void snapshotPreceededBySchemaChange()
                                     throws Exception

Throws:

Exception

snapshotWithRegexDataCollections

public void snapshotWithRegexDataCollections()
                                      throws Exception

Throws:

Exception

snapshotWithDuplicateDataCollections

@FixFor("DBZ-6945")
public void snapshotWithDuplicateDataCollections()
                                          throws Exception

Throws:

Exception

stopCurrentIncrementalSnapshotWithoutCollectionsAndTakeNewNewIncrementalSnapshotAfterRestart

@FixFor("DBZ-4271")
public void stopCurrentIncrementalSnapshotWithoutCollectionsAndTakeNewNewIncrementalSnapshotAfterRestart()
                                                                                                  throws Exception

Throws:

Exception

stopCurrentIncrementalSnapshotWithAllCollectionsAndTakeNewNewIncrementalSnapshotAfterRestart

@FixFor("DBZ-4271")
public void stopCurrentIncrementalSnapshotWithAllCollectionsAndTakeNewNewIncrementalSnapshotAfterRestart()
                                                                                                  throws Exception

Throws:

Exception

removeNotYetCapturedCollectionFromInProgressIncrementalSnapshot

@FixFor("DBZ-4271")
public void removeNotYetCapturedCollectionFromInProgressIncrementalSnapshot()
                                                                     throws Exception

Throws:

Exception

removeStartedCapturedCollectionFromInProgressIncrementalSnapshot

@FixFor("DBZ-4271")
public void removeStartedCapturedCollectionFromInProgressIncrementalSnapshot()
                                                                      throws Exception

Throws:

Exception

shouldSnapshotNewlyAddedTableToIncludeListAfterRestart

@FixFor("DBZ-4834")
public void shouldSnapshotNewlyAddedTableToIncludeListAfterRestart()
                                                            throws Exception

Throws:

Exception

testPauseDuringSnapshot

public void testPauseDuringSnapshot()
                             throws Exception

Throws:

Exception

snapshotWithAdditionalCondition

public void snapshotWithAdditionalCondition()
                                     throws Exception

Throws:

Exception

snapshotWithNewAdditionalConditionsField

public void snapshotWithNewAdditionalConditionsField()
                                              throws Exception

Throws:

Exception

shouldExecuteRegularSnapshotWhenAdditionalConditionEmpty

public void shouldExecuteRegularSnapshotWhenAdditionalConditionEmpty()
                                                              throws Exception

Throws:

Exception

snapshotWithAdditionalConditionWithRestart

public void snapshotWithAdditionalConditionWithRestart()
                                                throws Exception

Throws:

Exception

snapshotWithSurrogateKey

public void snapshotWithSurrogateKey()
                              throws Exception

Throws:

Exception

snapshotWithAdditionalConditionWithSurrogateKey

public void snapshotWithAdditionalConditionWithSurrogateKey()
                                                     throws Exception

Throws:

Exception

testNotification

public void testNotification()
                      throws Exception

Throws:

Exception

insertInsertWatermarkingStrategy

public void insertInsertWatermarkingStrategy()
                                      throws Exception

Throws:

Exception

insertDeleteWatermarkingStrategy

public void insertDeleteWatermarkingStrategy()
                                      throws Exception

Throws:

Exception

assertOpenCloseEventCount

private void assertOpenCloseEventCount(JdbcConnection.ResultSetConsumer consumer)
                                throws SQLException

Throws:

SQLException

defaultIncrementalSnapshotChunkSize

protected int defaultIncrementalSnapshotChunkSize()

incrementalSnapshotCompleted

private static BiPredicate<Integer,org.apache.kafka.connect.source.SourceRecord> incrementalSnapshotCompleted()

assertCorrectIncrementalSnapshotNotification

private void assertCorrectIncrementalSnapshotNotification(List<org.apache.kafka.connect.source.SourceRecord> notifications)

sendAdHocSnapshotSignalAndWait

protected void sendAdHocSnapshotSignalAndWait(String... collectionIds)
                                       throws Exception

Throws:

Exception

sendAdHocSnapshotStopSignalAndWait

protected void sendAdHocSnapshotStopSignalAndWait(String... collectionIds)
                                           throws Exception

Throws:

Exception

consumeAnyRemainingIncrementalSnapshotEventsAndCheckForStopMessage

protected boolean consumeAnyRemainingIncrementalSnapshotEventsAndCheckForStopMessage(LogInterceptor interceptor,
 String stopMessage)
                                                                              throws Exception

Throws:

Exception