Package io.debezium.pipeline.source.snapshot.incremental

Class AbstractSnapshotTest<T extends org.apache.kafka.connect.source.SourceConnector>

java.lang.Object

io.debezium.embedded.AbstractConnectorTest

io.debezium.embedded.async.AbstractAsyncEngineConnectorTest

io.debezium.pipeline.source.snapshot.incremental.AbstractSnapshotTest<T>

All Implemented Interfaces:

Testing

Direct Known Subclasses:

AbstractBlockingSnapshotTest, AbstractIncrementalSnapshotTest

public abstract class AbstractSnapshotTest<T extends org.apache.kafka.connect.source.SourceConnector>
extends AbstractAsyncEngineConnectorTest

Nested Class Summary

Nested classes/interfaces inherited from class io.debezium.embedded.AbstractConnectorTest

AbstractConnectorTest.SourceRecords

Nested classes/interfaces inherited from interface io.debezium.util.Testing

Testing.Debug, Testing.Files, Testing.InterruptableFunction, Testing.Network, Testing.Print, Testing.Timer

Field Summary

Fields

Modifier and Type	Field	Description
private static final int	MAXIMUM_NO_RECORDS_CONSUMES
protected static final int	PARTITION_NO
protected static final int	ROW_COUNT
protected static final Path	SCHEMA_HISTORY_PATH
protected static final String	SERVER_NAME
protected final Path	signalsFile

Fields inherited from class io.debezium.embedded.AbstractConnectorTest

consumedLines, engine, isEngineRunning, logger, logTestName, OFFSET_STORE_PATH, pollTimeoutInMs, skipTestRule

Constructor Summary

Constructors

Constructor	Description
AbstractSnapshotTest()

Method Summary

Modifier and Type	Method	Description
protected String	alterTableAddColumnStatement(String tableName)
protected String	alterTableDropColumnStatement(String tableName)
private static String	buildAdditionalConditions(Map<String,String> additionalConditions)
protected abstract Configuration.Builder	config()
protected abstract String	connector()
protected abstract Class<T>	connectorClass()
protected Map<Integer,Integer>	consumeMixedWithIncrementalSnapshot(int recordCount)
protected Map<Integer,Integer>	consumeMixedWithIncrementalSnapshot(int recordCount, String topicName)
protected <V> Map<Integer,V>	consumeMixedWithIncrementalSnapshot(int recordCount, Function<org.apache.kafka.connect.source.SourceRecord,V> valueConverter, Predicate<Map.Entry<Integer,V>> dataCompleted, Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer, String topicName)
protected Map<Integer,Integer>	consumeMixedWithIncrementalSnapshot(int recordCount, Predicate<Map.Entry<Integer,Integer>> dataCompleted, Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
protected <V> Map<Integer,V>	consumeMixedWithIncrementalSnapshot(int recordCount, Predicate<Map.Entry<Integer,V>> dataCompleted, Function<org.apache.kafka.connect.data.Struct,Integer> idCalculator, Function<org.apache.kafka.connect.source.SourceRecord,V> valueConverter, String topicName, Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
protected <V> Map<Integer,V>	consumeMixedWithIncrementalSnapshot(int recordCount, Predicate<Map.Entry<Integer,V>> dataCompleted, Function<org.apache.kafka.connect.data.Struct,Integer> idCalculator, Function<org.apache.kafka.connect.source.SourceRecord,V> valueConverter, String topicName, Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer, boolean assertRecords)
protected Map<Integer,org.apache.kafka.connect.source.SourceRecord>	consumeRecordsMixedWithIncrementalSnapshot(int recordCount)
protected Map<Integer,org.apache.kafka.connect.source.SourceRecord>	consumeRecordsMixedWithIncrementalSnapshot(int recordCount, Predicate<Map.Entry<Integer,org.apache.kafka.connect.source.SourceRecord>> dataCompleted, Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
protected String	database()
protected abstract JdbcConnection	databaseConnection()
protected int	getMaximumEnqueuedRecordCount()	Get the maximum number of messages that can be obtained from the connector and held in-memory before they are consumed by test methods using AbstractConnectorTest.consumeRecord(), AbstractConnectorTest.consumeRecords(int), or AbstractConnectorTest.consumeRecords(int, Consumer).
protected Function<org.apache.kafka.connect.data.Struct,Integer>	getRecordValue()
protected abstract Configuration.Builder	mutableConfig(boolean signalTableOnly, boolean storeOnlyCapturedDdl)
protected String	pkFieldName()
protected void	populate4PkTable(JdbcConnection connection, String tableName)
protected void	populateTable()
protected void	populateTable(JdbcConnection connection)
protected void	populateTable(JdbcConnection connection, String tableName)
protected void	populateTable(String table)
protected void	populateTables()
protected void	populateTables(JdbcConnection connection)
protected void	populateTableWithSpecificValue(int startRow, int count, int value)
private void	populateTableWithSpecificValue(JdbcConnection connection, String tableName, int startRow, int count, int value)
protected void	sendAdHocSnapshotSignal(String... dataCollectionIds)
protected void	sendAdHocSnapshotSignalWithAdditionalConditionsWithSurrogateKey(Map<String,String> additionalConditions, String surrogateKey, io.debezium.pipeline.signal.actions.AbstractSnapshotSignal.SnapshotType snapshotType, String... dataCollectionIds)
protected void	sendAdHocSnapshotSignalWithAdditionalConditionsWithSurrogateKey(Map<String,String> additionalConditions, String surrogateKey, String... dataCollectionIds)
protected void	sendAdHocSnapshotSignalWithAdditionalConditionWithSurrogateKey(String additionalCondition, String surrogateKey, io.debezium.pipeline.signal.actions.AbstractSnapshotSignal.SnapshotType snapshotType, String... dataCollectionIds)
protected void	sendAdHocSnapshotSignalWithAdditionalConditionWithSurrogateKey(String additionalCondition, String surrogateKey, String... dataCollectionIds)
protected void	sendExecuteSnapshotFileSignal(String fullTableNames)
protected void	sendExecuteSnapshotFileSignal(String fullTableNames, String type, Path signalFile)
protected abstract String	server()
protected abstract String	signalTableName()
protected String	signalTableNameSanitized()
protected void	startConnector()
protected void	startConnector(io.debezium.engine.DebeziumEngine.CompletionCallback callback)
protected void	startConnector(Function<Configuration.Builder,Configuration.Builder> custConfig)
protected void	startConnector(Function<Configuration.Builder,Configuration.Builder> custConfig, io.debezium.engine.DebeziumEngine.CompletionCallback callback, boolean expectNoRecords)
protected void	startConnectorWithSnapshot(Function<Configuration.Builder,Configuration.Builder> custConfig)
protected String	tableDataCollectionId()
protected List<String>	tableDataCollectionIds()
protected abstract String	tableName()
protected abstract List<String>	tableNames()
protected String	task()
protected abstract String	topicName()
protected abstract List<String>	topicNames()
protected String	valueFieldName()
protected void	waitForCdcTransactionPropagation(int expectedTransactions)
protected void	waitForConnectorToStart()

Methods inherited from class io.debezium.embedded.async.AbstractAsyncEngineConnectorTest

createEngine, createEngineBuilder

Methods inherited from class io.debezium.embedded.AbstractConnectorTest

assertBeginTransaction, assertConfigurationErrors, assertConfigurationErrors, assertConfigurationErrors, assertConnectorIsRunning, assertConnectorNotRunning, assertDelete, assertEndTransaction, assertEngineIsRunning, assertHasNoSourceQuery, assertInsert, assertKey, assertNoConfigurationErrors, assertNoRecordsToConsume, assertOffset, assertOffset, assertOnlyTransactionRecordsToConsume, assertRecordTransactionMetadata, assertSchemaMatchesStruct, assertSchemaMatchesStruct, assertSourceQuery, assertTombstone, assertTombstone, assertUpdate, assertValueField, configValue, consumeAvailableRecords, consumeAvailableRecordsByTopic, consumeDmlRecordsByTopic, consumeDmlRecordsByTopic, consumeDmlRecordsByTopic, consumeRecord, consumeRecords, consumeRecords, consumeRecords, consumeRecordsButSkipUntil, consumeRecordsByTopic, consumeRecordsByTopic, consumeRecordsByTopic, consumeRecordsByTopicUntil, consumeRecordsUntil, debug, getConsumer, getSnapshotMetricsObjectName, getSnapshotMetricsObjectName, getSnapshotMetricsObjectName, getStreamingMetricsObjectName, getStreamingMetricsObjectName, getStreamingMetricsObjectName, getStreamingMetricsObjectName, getStreamingNamespace, initializeConnectorTestFramework, isStreamingRunning, isStreamingRunning, isStreamingRunning, isStreamingRunning, isTransactionRecord, loggingCompletion, print, readLastCommittedOffset, readLastCommittedOffsets, setConsumeTimeout, skipAvroValidation, start, start, start, start, start, start, start, startAndConsumeTillEnd, startAndConsumeTillEnd, stopConnector, stopConnector, storeOffsets, validate, waitForAvailableRecords, waitForConnectorShutdown, waitForEngineShutdown, waitForSnapshotToBeCompleted, waitForSnapshotToBeCompleted, waitForSnapshotWithCustomMetricsToBeCompleted, waitForStreamingRunning, waitForStreamingRunning, waitForStreamingRunning, waitForStreamingWithCustomMetricsToStart, waitTimeForEngine, waitTimeForRecords, waitTimeForRecordsAfterNulls

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface io.debezium.util.Testing

once, once, resetBeforeEachTest, time, time

Field Details

ROW_COUNT

protected static final int ROW_COUNT

See Also:

• Constant Field Values

SCHEMA_HISTORY_PATH

protected static final Path SCHEMA_HISTORY_PATH

PARTITION_NO

protected static final int PARTITION_NO

See Also:

• Constant Field Values

SERVER_NAME

protected static final String SERVER_NAME

See Also:

• Constant Field Values

MAXIMUM_NO_RECORDS_CONSUMES

private static final int MAXIMUM_NO_RECORDS_CONSUMES

See Also:

• Constant Field Values

signalsFile

protected final Path signalsFile

Constructor Details

AbstractSnapshotTest

public AbstractSnapshotTest()

Method Details

connectorClass

protected abstract Class<T> connectorClass()

databaseConnection

protected abstract JdbcConnection databaseConnection()

topicName

protected abstract String topicName()

tableName

protected abstract String tableName()

topicNames

protected abstract List<String> topicNames()

tableNames

protected abstract List<String> tableNames()

signalTableName

protected abstract String signalTableName()

signalTableNameSanitized

protected String signalTableNameSanitized()

config

protected abstract Configuration.Builder config()

mutableConfig

protected abstract Configuration.Builder mutableConfig(boolean signalTableOnly,
 boolean storeOnlyCapturedDdl)

connector

protected abstract String connector()

server

protected abstract String server()

task

protected String task()

database

protected String database()

waitForCdcTransactionPropagation

protected void waitForCdcTransactionPropagation(int expectedTransactions)
                                         throws Exception

Throws:

Exception

alterTableAddColumnStatement

protected String alterTableAddColumnStatement(String tableName)

alterTableDropColumnStatement

protected String alterTableDropColumnStatement(String tableName)

tableDataCollectionId

protected String tableDataCollectionId()

tableDataCollectionIds

protected List<String> tableDataCollectionIds()

populateTable

protected void populateTable(JdbcConnection connection,
 String tableName)
                      throws SQLException

Throws:

SQLException

populateTable

protected void populateTable(JdbcConnection connection)
                      throws SQLException

Throws:

SQLException

populateTables

protected void populateTables(JdbcConnection connection)
                       throws SQLException

Throws:

SQLException

populateTable

protected void populateTable()
                      throws SQLException

Throws:

SQLException

populateTable

protected void populateTable(String table)
                      throws SQLException

Throws:

SQLException

populateTableWithSpecificValue

protected void populateTableWithSpecificValue(int startRow,
 int count,
 int value)
                                       throws SQLException

Throws:

SQLException

populateTableWithSpecificValue

private void populateTableWithSpecificValue(JdbcConnection connection,
 String tableName,
 int startRow,
 int count,
 int value)
                                     throws SQLException

Throws:

SQLException

populateTables

protected void populateTables()
                       throws SQLException

Throws:

SQLException

populate4PkTable

protected void populate4PkTable(JdbcConnection connection,
 String tableName)
                         throws SQLException

Throws:

SQLException

consumeMixedWithIncrementalSnapshot

protected Map<Integer,Integer> consumeMixedWithIncrementalSnapshot(int recordCount)
                                                            throws InterruptedException

Throws:

InterruptedException

consumeMixedWithIncrementalSnapshot

protected Map<Integer,Integer> consumeMixedWithIncrementalSnapshot(int recordCount,
 String topicName)
                                                            throws InterruptedException

Throws:

InterruptedException

consumeMixedWithIncrementalSnapshot

protected <V> Map<Integer,V> consumeMixedWithIncrementalSnapshot(int recordCount,
 Function<org.apache.kafka.connect.source.SourceRecord,V> valueConverter,
 Predicate<Map.Entry<Integer,V>> dataCompleted,
 Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer,
 String topicName)
                                                          throws InterruptedException

Throws:

InterruptedException

consumeMixedWithIncrementalSnapshot

protected <V> Map<Integer,V> consumeMixedWithIncrementalSnapshot(int recordCount,
 Predicate<Map.Entry<Integer,V>> dataCompleted,
 Function<org.apache.kafka.connect.data.Struct,Integer> idCalculator,
 Function<org.apache.kafka.connect.source.SourceRecord,V> valueConverter,
 String topicName,
 Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
                                                          throws InterruptedException

Throws:

InterruptedException

consumeMixedWithIncrementalSnapshot

protected <V> Map<Integer,V> consumeMixedWithIncrementalSnapshot(int recordCount,
 Predicate<Map.Entry<Integer,V>> dataCompleted,
 Function<org.apache.kafka.connect.data.Struct,Integer> idCalculator,
 Function<org.apache.kafka.connect.source.SourceRecord,V> valueConverter,
 String topicName,
 Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer,
 boolean assertRecords)
                                                          throws InterruptedException

Throws:

InterruptedException

consumeRecordsMixedWithIncrementalSnapshot

protected Map<Integer,org.apache.kafka.connect.source.SourceRecord> consumeRecordsMixedWithIncrementalSnapshot(int recordCount)
                                                                                                        throws InterruptedException

Throws:

InterruptedException

consumeMixedWithIncrementalSnapshot

protected Map<Integer,Integer> consumeMixedWithIncrementalSnapshot(int recordCount,
 Predicate<Map.Entry<Integer,Integer>> dataCompleted,
 Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
                                                            throws InterruptedException

Throws:

InterruptedException

consumeRecordsMixedWithIncrementalSnapshot

protected Map<Integer,org.apache.kafka.connect.source.SourceRecord> consumeRecordsMixedWithIncrementalSnapshot(int recordCount,
 Predicate<Map.Entry<Integer,org.apache.kafka.connect.source.SourceRecord>> dataCompleted,
 Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
                                                                                                        throws InterruptedException

Throws:

InterruptedException

valueFieldName

protected String valueFieldName()

pkFieldName

protected String pkFieldName()

startConnector

protected void startConnector(io.debezium.engine.DebeziumEngine.CompletionCallback callback)

startConnector

protected void startConnector(Function<Configuration.Builder,Configuration.Builder> custConfig)

startConnector

protected void startConnector(Function<Configuration.Builder,Configuration.Builder> custConfig,
 io.debezium.engine.DebeziumEngine.CompletionCallback callback,
 boolean expectNoRecords)

startConnectorWithSnapshot

protected void startConnectorWithSnapshot(Function<Configuration.Builder,Configuration.Builder> custConfig)

startConnector

protected void startConnector()

waitForConnectorToStart

protected void waitForConnectorToStart()

getRecordValue

protected Function<org.apache.kafka.connect.data.Struct,Integer> getRecordValue()

getMaximumEnqueuedRecordCount

protected int getMaximumEnqueuedRecordCount()

Description copied from class: AbstractConnectorTest

Get the maximum number of messages that can be obtained from the connector and held in-memory before they are consumed by test methods using AbstractConnectorTest.consumeRecord(), AbstractConnectorTest.consumeRecords(int), or AbstractConnectorTest.consumeRecords(int, Consumer).

By default this method return 100.

Overrides:

getMaximumEnqueuedRecordCount in class AbstractConnectorTest

Returns:

the maximum number of records that can be enqueued

sendExecuteSnapshotFileSignal

protected void sendExecuteSnapshotFileSignal(String fullTableNames)
                                      throws IOException

Throws:

IOException

sendExecuteSnapshotFileSignal

protected void sendExecuteSnapshotFileSignal(String fullTableNames,
 String type,
 Path signalFile)
                                      throws IOException

Throws:

IOException

sendAdHocSnapshotSignal

protected void sendAdHocSnapshotSignal(String... dataCollectionIds)
                                throws SQLException

Throws:

SQLException

sendAdHocSnapshotSignalWithAdditionalConditionWithSurrogateKey

protected void sendAdHocSnapshotSignalWithAdditionalConditionWithSurrogateKey(String additionalCondition,
 String surrogateKey,
 String... dataCollectionIds)

sendAdHocSnapshotSignalWithAdditionalConditionsWithSurrogateKey

protected void sendAdHocSnapshotSignalWithAdditionalConditionsWithSurrogateKey(Map<String,String> additionalConditions,
 String surrogateKey,
 String... dataCollectionIds)

sendAdHocSnapshotSignalWithAdditionalConditionWithSurrogateKey

protected void sendAdHocSnapshotSignalWithAdditionalConditionWithSurrogateKey(String additionalCondition,
 String surrogateKey,
 io.debezium.pipeline.signal.actions.AbstractSnapshotSignal.SnapshotType snapshotType,
 String... dataCollectionIds)

sendAdHocSnapshotSignalWithAdditionalConditionsWithSurrogateKey

protected void sendAdHocSnapshotSignalWithAdditionalConditionsWithSurrogateKey(Map<String,String> additionalConditions,
 String surrogateKey,
 io.debezium.pipeline.signal.actions.AbstractSnapshotSignal.SnapshotType snapshotType,
 String... dataCollectionIds)

buildAdditionalConditions

private static String buildAdditionalConditions(Map<String,String> additionalConditions)