Package io.debezium.connector.mongodb

Class IncrementalSnapshotIT

java.lang.Object

io.debezium.embedded.AbstractConnectorTest

io.debezium.connector.mongodb.AbstractMongoConnectorIT

io.debezium.connector.mongodb.IncrementalSnapshotIT

All Implemented Interfaces:

Testing

public class IncrementalSnapshotIT
extends AbstractMongoConnectorIT

Test to verify incremental snapshotting for MongoDB.

Author:

Jiri Pechanec

Nested Class Summary

Nested classes/interfaces inherited from class io.debezium.embedded.AbstractConnectorTest

AbstractConnectorTest.SourceRecords

Nested classes/interfaces inherited from interface io.debezium.util.Testing

Testing.Debug, Testing.Files, Testing.InterruptableFunction, Testing.Network, Testing.Print, Testing.Timer

Field Summary

Fields

Modifier and Type	Field	Description
private static final String	COLLECTION_NAME
private static final String	COLLECTION2_NAME
private static final String	DATABASE_NAME
protected static final Path	DB_HISTORY_PATH
private static final String	DOCUMENT_ID
private static final String	FULL_COLLECTION_NAME
private static final String	FULL_COLLECTION2_NAME
private static final int	MAXIMUM_NO_RECORDS_CONSUMES
protected static final int	ROW_COUNT
private static final String	SIGNAL_COLLECTION_NAME

Fields inherited from class io.debezium.connector.mongodb.AbstractMongoConnectorIT

config, context, logInterceptor

Fields inherited from class io.debezium.embedded.AbstractConnectorTest

consumedLines, engine, logger, logTestName, OFFSET_STORE_PATH, pollTimeoutInMs, skipTestRule

Constructor Summary

Constructors

Constructor	Description
IncrementalSnapshotIT()

Method Summary

Modifier and Type	Method	Description
void	after()
void	before()
protected Configuration.Builder	config()
protected Class<io.debezium.connector.mongodb.MongoDbConnector>	connectorClass()
protected boolean	consumeAnyRemainingIncrementalSnapshotEventsAndCheckForStopMessage(LogInterceptor interceptor, String stopMessage)
protected Map<Integer,Integer>	consumeMixedWithIncrementalSnapshot(int recordCount)
protected Map<Integer,Integer>	consumeMixedWithIncrementalSnapshot(int recordCount, String topicName)
protected <V> Map<Integer,V>	consumeMixedWithIncrementalSnapshot(int recordCount, Function<org.apache.kafka.connect.source.SourceRecord,V> valueConverter, Predicate<Map.Entry<Integer,V>> dataCompleted, Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer, String topicName)
protected Map<Integer,Integer>	consumeMixedWithIncrementalSnapshot(int recordCount, Predicate<Map.Entry<Integer,Integer>> dataCompleted, Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
protected <V> Map<Integer,V>	consumeMixedWithIncrementalSnapshot(int recordCount, Predicate<Map.Entry<Integer,V>> dataCompleted, Function<org.apache.kafka.connect.data.Struct,Integer> idCalculator, Function<org.apache.kafka.connect.source.SourceRecord,V> valueConverter, String topicName, Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
protected Map<Integer,org.apache.kafka.connect.source.SourceRecord>	consumeRecordsMixedWithIncrementalSnapshot(int recordCount)
protected Map<Integer,org.apache.kafka.connect.source.SourceRecord>	consumeRecordsMixedWithIncrementalSnapshot(int recordCount, Predicate<Map.Entry<Integer,org.apache.kafka.connect.source.SourceRecord>> dataCompleted, Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
protected String	dataCollectionName()
protected List<String>	dataCollectionNames()
protected Integer	extractFieldValue(org.apache.kafka.connect.source.SourceRecord record)
protected String	fullDataCollectionName()
protected List<String>	fullDataCollectionNames()
protected int	getMaximumEnqueuedRecordCount()
protected void	insertAdditionalData()
protected void	insertAdditionalData(String collectionName)
void	inserts()
void	invalidTablesInTheList()
protected String	pkFieldName()
protected void	populateDataCollection()
protected void	populateDataCollection(io.debezium.connector.mongodb.ConnectionContext.MongoPrimary connection)
protected void	populateDataCollection(io.debezium.connector.mongodb.ConnectionContext.MongoPrimary connection, String dataCollectionName)
protected void	populateDataCollections()
protected void	populateDataCollections(io.debezium.connector.mongodb.ConnectionContext.MongoPrimary connection)
void	removeNotYetCapturedCollectionFromInProgressIncrementalSnapshot()
void	removeStartedCapturedCollectionFromInProgressIncrementalSnapshot()
protected void	sendAdHocSnapshotSignal()
protected void	sendAdHocSnapshotSignal(String... dataCollectionIds)
protected void	sendAdHocSnapshotSignalAndWait(String... collectionIds)
protected void	sendAdHocSnapshotStopSignal(String... dataCollectionIds)
protected void	sendAdHocSnapshotStopSignalAndWait(String... collectionIds)
void	snapshotOnly()
void	snapshotOnlyWithRestart()
protected void	startConnector()
protected void	startConnector(io.debezium.engine.DebeziumEngine.CompletionCallback callback)
protected void	startConnector(Function<Configuration.Builder,Configuration.Builder> custConfig)
protected void	startConnector(Function<Configuration.Builder,Configuration.Builder> custConfig, io.debezium.engine.DebeziumEngine.CompletionCallback callback)
void	stopCurrentIncrementalSnapshotWithAllCollectionsAndTakeNewNewIncrementalSnapshotAfterRestart()
void	stopCurrentIncrementalSnapshotWithoutCollectionsAndTakeNewNewIncrementalSnapshotAfterRestart()
protected String	topicName()
protected List<String>	topicNames()
protected void	updateData(int batchSize)
void	updates()
void	updatesLargeChunk()
void	updatesWithRestart()
protected String	valueFieldName()
protected void	waitForConnectorToStart()

Methods inherited from class io.debezium.connector.mongodb.AbstractMongoConnectorIT

afterEach, beforeEach, connectionErrorHandler, deleteDocuments, dropAndInsertDocuments, getSnapshotMetricsObjectName, getSnapshotMetricsObjectName, getStreamingMetricsObjectName, getStreamingMetricsObjectName, insertDocuments, insertDocumentsInTx, loadTestDocuments, primary, storeDocuments, storeDocuments, updateDocument, updateDocumentsInTx, waitForSnapshotToBeCompleted, waitForSnapshotToBeCompleted, waitForStreamingRunning, waitForStreamingRunning

Methods inherited from class io.debezium.embedded.AbstractConnectorTest

assertBeginTransaction, assertConfigurationErrors, assertConfigurationErrors, assertConfigurationErrors, assertConnectorIsRunning, assertConnectorNotRunning, assertDelete, assertEndTransaction, assertEngineIsRunning, assertHasNoSourceQuery, assertInsert, assertKey, assertNoConfigurationErrors, assertNoRecordsToConsume, assertOffset, assertOffset, assertOnlyTransactionRecordsToConsume, assertRecordTransactionMetadata, assertSchemaMatchesStruct, assertSchemaMatchesStruct, assertSourceQuery, assertTombstone, assertTombstone, assertUpdate, assertValueField, configValue, consumeAvailableRecords, consumeDmlRecordsByTopic, consumeDmlRecordsByTopic, consumeDmlRecordsByTopic, consumeRecord, consumeRecords, consumeRecords, consumeRecords, consumeRecordsByTopic, consumeRecordsByTopic, consumeRecordsByTopic, consumeRecordsByTopicUntil, consumeRecordsUntil, debug, getConsumer, getStreamingMetricsObjectName, getStreamingNamespace, initializeConnectorTestFramework, isStreamingRunning, isStreamingRunning, isTransactionRecord, loggingCompletion, print, readLastCommittedOffset, readLastCommittedOffsets, setConsumeTimeout, skipAvroValidation, start, start, start, start, start, start, start, startAndConsumeTillEnd, startAndConsumeTillEnd, stopConnector, stopConnector, validate, waitForAvailableRecords, waitForConnectorShutdown, waitForStreamingRunning, waitTimeForRecords, waitTimeForRecordsAfterNulls

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface io.debezium.util.Testing

once, once, resetBeforeEachTest, time, time

Field Details

ROW_COUNT

protected static final int ROW_COUNT

See Also:

• Constant Field Values

MAXIMUM_NO_RECORDS_CONSUMES

private static final int MAXIMUM_NO_RECORDS_CONSUMES

See Also:

• Constant Field Values

DATABASE_NAME

private static final String DATABASE_NAME

See Also:

• Constant Field Values

COLLECTION_NAME

private static final String COLLECTION_NAME

See Also:

• Constant Field Values

COLLECTION2_NAME

private static final String COLLECTION2_NAME

See Also:

• Constant Field Values

SIGNAL_COLLECTION_NAME

private static final String SIGNAL_COLLECTION_NAME

See Also:

• Constant Field Values

FULL_COLLECTION_NAME

private static final String FULL_COLLECTION_NAME

See Also:

• Constant Field Values

FULL_COLLECTION2_NAME

private static final String FULL_COLLECTION2_NAME

See Also:

• Constant Field Values

DOCUMENT_ID

private static final String DOCUMENT_ID

See Also:

• Constant Field Values

DB_HISTORY_PATH

protected static final Path DB_HISTORY_PATH

Constructor Details

IncrementalSnapshotIT

public IncrementalSnapshotIT()

Method Details

before

public void before()

after

public void after()

connectorClass

protected Class<io.debezium.connector.mongodb.MongoDbConnector> connectorClass()

config

protected Configuration.Builder config()

dataCollectionName

protected String dataCollectionName()

dataCollectionNames

protected List<String> dataCollectionNames()

fullDataCollectionName

protected String fullDataCollectionName()

fullDataCollectionNames

protected List<String> fullDataCollectionNames()

topicName

protected String topicName()

topicNames

protected List<String> topicNames()

populateDataCollection

protected void populateDataCollection(io.debezium.connector.mongodb.ConnectionContext.MongoPrimary connection,
 String dataCollectionName)

populateDataCollection

protected void populateDataCollection(io.debezium.connector.mongodb.ConnectionContext.MongoPrimary connection)

populateDataCollections

protected void populateDataCollections(io.debezium.connector.mongodb.ConnectionContext.MongoPrimary connection)

populateDataCollection

protected void populateDataCollection()

populateDataCollections

protected void populateDataCollections()

insertAdditionalData

protected void insertAdditionalData()

insertAdditionalData

protected void insertAdditionalData(String collectionName)

updateData

protected void updateData(int batchSize)

startConnector

protected void startConnector(io.debezium.engine.DebeziumEngine.CompletionCallback callback)

startConnector

protected void startConnector(Function<Configuration.Builder,Configuration.Builder> custConfig)

startConnector

protected void startConnector(Function<Configuration.Builder,Configuration.Builder> custConfig,
 io.debezium.engine.DebeziumEngine.CompletionCallback callback)

startConnector

protected void startConnector()

waitForConnectorToStart

protected void waitForConnectorToStart()

sendAdHocSnapshotSignal

protected void sendAdHocSnapshotSignal(String... dataCollectionIds)
                                throws SQLException

Throws:

SQLException

sendAdHocSnapshotStopSignal

protected void sendAdHocSnapshotStopSignal(String... dataCollectionIds)
                                    throws SQLException

Throws:

SQLException

sendAdHocSnapshotSignal

protected void sendAdHocSnapshotSignal()
                                throws SQLException

Throws:

SQLException

consumeMixedWithIncrementalSnapshot

protected Map<Integer,Integer> consumeMixedWithIncrementalSnapshot(int recordCount)
                                                            throws InterruptedException

Throws:

InterruptedException

consumeMixedWithIncrementalSnapshot

protected Map<Integer,Integer> consumeMixedWithIncrementalSnapshot(int recordCount,
 String topicName)
                                                            throws InterruptedException

Throws:

InterruptedException

extractFieldValue

protected Integer extractFieldValue(org.apache.kafka.connect.source.SourceRecord record)

consumeMixedWithIncrementalSnapshot

protected <V> Map<Integer,V> consumeMixedWithIncrementalSnapshot(int recordCount,
 Function<org.apache.kafka.connect.source.SourceRecord,V> valueConverter,
 Predicate<Map.Entry<Integer,V>> dataCompleted,
 Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer,
 String topicName)
                                                          throws InterruptedException

Throws:

InterruptedException

consumeMixedWithIncrementalSnapshot

protected <V> Map<Integer,V> consumeMixedWithIncrementalSnapshot(int recordCount,
 Predicate<Map.Entry<Integer,V>> dataCompleted,
 Function<org.apache.kafka.connect.data.Struct,Integer> idCalculator,
 Function<org.apache.kafka.connect.source.SourceRecord,V> valueConverter,
 String topicName,
 Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
                                                          throws InterruptedException

Throws:

InterruptedException

consumeRecordsMixedWithIncrementalSnapshot

protected Map<Integer,org.apache.kafka.connect.source.SourceRecord> consumeRecordsMixedWithIncrementalSnapshot(int recordCount)
                                                                                                        throws InterruptedException

Throws:

InterruptedException

consumeMixedWithIncrementalSnapshot

protected Map<Integer,Integer> consumeMixedWithIncrementalSnapshot(int recordCount,
 Predicate<Map.Entry<Integer,Integer>> dataCompleted,
 Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
                                                            throws InterruptedException

Throws:

InterruptedException

consumeRecordsMixedWithIncrementalSnapshot

protected Map<Integer,org.apache.kafka.connect.source.SourceRecord> consumeRecordsMixedWithIncrementalSnapshot(int recordCount,
 Predicate<Map.Entry<Integer,org.apache.kafka.connect.source.SourceRecord>> dataCompleted,
 Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
                                                                                                        throws InterruptedException

Throws:

InterruptedException

valueFieldName

protected String valueFieldName()

pkFieldName

protected String pkFieldName()

snapshotOnly

public void snapshotOnly()
                  throws Exception

Throws:

Exception

invalidTablesInTheList

public void invalidTablesInTheList()
                            throws Exception

Throws:

Exception

snapshotOnlyWithRestart

public void snapshotOnlyWithRestart()
                             throws Exception

Throws:

Exception

inserts

public void inserts()
             throws Exception

Throws:

Exception

updates

public void updates()
             throws Exception

Throws:

Exception

updatesWithRestart

public void updatesWithRestart()
                        throws Exception

Throws:

Exception

updatesLargeChunk

public void updatesLargeChunk()
                       throws Exception

Throws:

Exception

stopCurrentIncrementalSnapshotWithoutCollectionsAndTakeNewNewIncrementalSnapshotAfterRestart

@FixFor("DBZ-4271")
public void stopCurrentIncrementalSnapshotWithoutCollectionsAndTakeNewNewIncrementalSnapshotAfterRestart()
                                                                                                  throws Exception

Throws:

Exception

stopCurrentIncrementalSnapshotWithAllCollectionsAndTakeNewNewIncrementalSnapshotAfterRestart

@FixFor("DBZ-4271")
public void stopCurrentIncrementalSnapshotWithAllCollectionsAndTakeNewNewIncrementalSnapshotAfterRestart()
                                                                                                  throws Exception

Throws:

Exception

removeNotYetCapturedCollectionFromInProgressIncrementalSnapshot

@FixFor("DBZ-4271")
public void removeNotYetCapturedCollectionFromInProgressIncrementalSnapshot()
                                                                     throws Exception

Throws:

Exception

removeStartedCapturedCollectionFromInProgressIncrementalSnapshot

@FixFor("DBZ-4271")
public void removeStartedCapturedCollectionFromInProgressIncrementalSnapshot()
                                                                      throws Exception

Throws:

Exception

sendAdHocSnapshotSignalAndWait

protected void sendAdHocSnapshotSignalAndWait(String... collectionIds)
                                       throws Exception

Throws:

Exception

sendAdHocSnapshotStopSignalAndWait

protected void sendAdHocSnapshotStopSignalAndWait(String... collectionIds)
                                           throws Exception

Throws:

Exception

consumeAnyRemainingIncrementalSnapshotEventsAndCheckForStopMessage

protected boolean consumeAnyRemainingIncrementalSnapshotEventsAndCheckForStopMessage(LogInterceptor interceptor,
 String stopMessage)
                                                                              throws Exception

Throws:

Exception

getMaximumEnqueuedRecordCount

protected int getMaximumEnqueuedRecordCount()

Overrides:

getMaximumEnqueuedRecordCount in class AbstractConnectorTest