Package io.debezium.connector.mongodb

Class IncrementalSnapshotIT

java.lang.Object

io.debezium.embedded.AbstractConnectorTest

io.debezium.connector.mongodb.AbstractMongoConnectorIT

io.debezium.connector.mongodb.IncrementalSnapshotIT

All Implemented Interfaces:

Testing

Direct Known Subclasses:

MultiThreadIncrementalSnapshotIT

public class IncrementalSnapshotIT
extends AbstractMongoConnectorIT

Test to verify incremental snapshotting for MongoDB.

Author:

Jiri Pechanec

Nested Class Summary

Nested classes/interfaces inherited from class io.debezium.embedded.AbstractConnectorTest

AbstractConnectorTest.SourceRecords

Nested classes/interfaces inherited from interface io.debezium.util.Testing

Testing.Debug, Testing.Files, Testing.InterruptableFunction, Testing.Network, Testing.Print, Testing.Timer

Field Summary

Fields

Modifier and Type	Field	Description
private static final String	COLLECTION_NAME
private static final String	COLLECTION2_NAME
private static final String	DATABASE_NAME
private static final String	DOCUMENT_ID
private static final String	FULL_COLLECTION_NAME
private static final String	FULL_COLLECTION2_NAME
private static final int	MAXIMUM_NO_RECORDS_CONSUMES
protected static final int	ROW_COUNT
private static final String	SIGNAL_COLLECTION_NAME

Fields inherited from class io.debezium.connector.mongodb.AbstractMongoConnectorIT

config, context, logInterceptor, mongo

Fields inherited from class io.debezium.embedded.AbstractConnectorTest

consumedLines, engine, isEngineRunning, logger, logTestName, OFFSET_STORE_PATH, pollTimeoutInMs, skipTestRule

Constructor Summary

Constructors

Constructor	Description
IncrementalSnapshotIT()

Method Summary

Modifier and Type	Method	Description
void	after()
private void	assertCloseEventCount(Consumer<Long> consumer)
void	before()
protected Configuration.Builder	config()
protected Class<io.debezium.connector.mongodb.MongoDbConnector>	connectorClass()
protected boolean	consumeAnyRemainingIncrementalSnapshotEventsAndCheckForStopMessage(LogInterceptor interceptor, String stopMessage)
protected Map<Integer,Integer>	consumeMixedWithIncrementalSnapshot(int recordCount)
protected Map<Integer,Integer>	consumeMixedWithIncrementalSnapshot(int recordCount, String topicName)
protected <V> Map<Integer,V>	consumeMixedWithIncrementalSnapshot(int recordCount, Function<org.apache.kafka.connect.source.SourceRecord,V> valueConverter, Predicate<Map.Entry<Integer,V>> dataCompleted, Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer, String topicName)
protected Map<Integer,Integer>	consumeMixedWithIncrementalSnapshot(int recordCount, Predicate<Map.Entry<Integer,Integer>> dataCompleted, Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
protected <V, K> Map<K,V>	consumeMixedWithIncrementalSnapshot(int recordCount, Predicate<Map.Entry<K,V>> dataCompleted, Function<org.apache.kafka.connect.data.Struct,K> idCalculator, Function<org.apache.kafka.connect.source.SourceRecord,V> valueConverter, String topicName, Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
protected Map<Integer,org.apache.kafka.connect.source.SourceRecord>	consumeRecordsMixedWithIncrementalSnapshot(int recordCount)
protected Map<Integer,org.apache.kafka.connect.source.SourceRecord>	consumeRecordsMixedWithIncrementalSnapshot(int recordCount, Predicate<Map.Entry<Integer,org.apache.kafka.connect.source.SourceRecord>> dataCompleted, Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
protected String	dataCollectionName()
protected List<String>	dataCollectionNames()
protected Integer	extractFieldValue(org.apache.kafka.connect.source.SourceRecord record)
protected String	fullDataCollectionName()
protected List<String>	fullDataCollectionNames()
protected int	getMaximumEnqueuedRecordCount()
protected void	insertAdditionalData()
protected void	insertAdditionalData(String collectionName)
void	insertDeleteWatermarkingStrategy()
void	insertInsertWatermarkingStrategy()
void	inserts()
void	invalidTablesInTheList()
void	pauseDuringSnapshot()
protected String	pkFieldName()
protected void	populateDataCollection()
protected void	populateDataCollection(String dataCollectionName)
protected void	populateDataCollections()
void	removeNotYetCapturedCollectionFromInProgressIncrementalSnapshot()
void	removeStartedCapturedCollectionFromInProgressIncrementalSnapshot()
protected void	sendAdHocSnapshotSignal()
protected void	sendAdHocSnapshotSignal(String... dataCollectionIds)
protected void	sendAdHocSnapshotSignalAndWait(String... collectionIds)
protected void	sendAdHocSnapshotStopSignal(String... dataCollectionIds)
protected void	sendAdHocSnapshotStopSignalAndWait(String... collectionIds)
protected void	sendPauseSignal()
protected void	sendResumeSignal()
void	shouldStreamWithDatabaseIncludeList()
private <K> void	snapshotOnly(K initialId, Function<K,K> idGenerator)
void	snapshotOnlyDecimal128()
void	snapshotOnlyDouble()
void	snapshotOnlyInt32()
void	snapshotOnlyObjectId()
void	snapshotOnlyString()
void	snapshotOnlyWithInt64()
void	snapshotOnlyWithRestart()
protected void	startConnector()
protected void	startConnector(Function<Configuration.Builder,Configuration.Builder> custConfig)
protected void	startConnector(Function<Configuration.Builder,Configuration.Builder> custConfig, io.debezium.engine.DebeziumEngine.CompletionCallback callback)
void	stopCurrentIncrementalSnapshotWithAllCollectionsAndTakeNewNewIncrementalSnapshotAfterRestart()
void	stopCurrentIncrementalSnapshotWithoutCollectionsAndTakeNewNewIncrementalSnapshotAfterRestart()
protected String	topicName()
protected List<String>	topicNames()
protected void	updateData(int batchSize)
void	updates()
void	updatesLargeChunk()
void	updatesWithRestart()
protected String	valueFieldName()
protected void	waitForConnectorToStart()

Methods inherited from class io.debezium.connector.mongodb.AbstractMongoConnectorIT

afterAll, afterEach, beforeAll, beforeEach, connect, deleteDocuments, dropAndInsertDocuments, getSnapshotMetricsObjectName, getSnapshotMetricsObjectName, getStreamingMetricsObjectName, getStreamingMetricsObjectName, insertDocuments, insertDocumentsInTx, loadTestDocuments, storeDocuments, storeDocuments, updateDocument, updateDocumentsInTx, waitForSnapshotToBeCompleted, waitForSnapshotToBeCompleted, waitForStreamingRunning, waitForStreamingRunning

Methods inherited from class io.debezium.embedded.AbstractConnectorTest

assertBeginTransaction, assertConfigurationErrors, assertConfigurationErrors, assertConfigurationErrors, assertConnectorIsRunning, assertConnectorNotRunning, assertDelete, assertEndTransaction, assertEngineIsRunning, assertHasNoSourceQuery, assertInsert, assertKey, assertNoConfigurationErrors, assertNoRecordsToConsume, assertOffset, assertOffset, assertOnlyTransactionRecordsToConsume, assertRecordTransactionMetadata, assertSchemaMatchesStruct, assertSchemaMatchesStruct, assertSourceQuery, assertTombstone, assertTombstone, assertUpdate, assertValueField, configValue, consumeAvailableRecords, consumeAvailableRecordsByTopic, consumeDmlRecordsByTopic, consumeDmlRecordsByTopic, consumeDmlRecordsByTopic, consumeRecord, consumeRecords, consumeRecords, consumeRecords, consumeRecordsButSkipUntil, consumeRecordsByTopic, consumeRecordsByTopic, consumeRecordsByTopic, consumeRecordsByTopicUntil, consumeRecordsUntil, debug, getConsumer, getSnapshotMetricsObjectName, getSnapshotMetricsObjectName, getStreamingMetricsObjectName, getStreamingMetricsObjectName, getStreamingMetricsObjectName, getStreamingNamespace, initializeConnectorTestFramework, isStreamingRunning, isStreamingRunning, isStreamingRunning, isStreamingRunning, isTransactionRecord, loggingCompletion, print, readLastCommittedOffset, readLastCommittedOffsets, setConsumeTimeout, skipAvroValidation, start, start, start, start, start, start, start, startAndConsumeTillEnd, startAndConsumeTillEnd, stopConnector, stopConnector, validate, waitForAvailableRecords, waitForConnectorShutdown, waitForSnapshotToBeCompleted, waitForSnapshotWithCustomMetricsToBeCompleted, waitForStreamingRunning, waitForStreamingRunning, waitForStreamingWithCustomMetricsToStart, waitTimeForRecords, waitTimeForRecordsAfterNulls

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface io.debezium.util.Testing

once, once, resetBeforeEachTest, time, time

Field Details

ROW_COUNT

protected static final int ROW_COUNT

See Also:

• Constant Field Values

MAXIMUM_NO_RECORDS_CONSUMES

private static final int MAXIMUM_NO_RECORDS_CONSUMES

See Also:

• Constant Field Values

DATABASE_NAME

private static final String DATABASE_NAME

See Also:

• Constant Field Values

COLLECTION_NAME

private static final String COLLECTION_NAME

See Also:

• Constant Field Values

COLLECTION2_NAME

private static final String COLLECTION2_NAME

See Also:

• Constant Field Values

SIGNAL_COLLECTION_NAME

private static final String SIGNAL_COLLECTION_NAME

See Also:

• Constant Field Values

FULL_COLLECTION_NAME

private static final String FULL_COLLECTION_NAME

See Also:

• Constant Field Values

FULL_COLLECTION2_NAME

private static final String FULL_COLLECTION2_NAME

See Also:

• Constant Field Values

DOCUMENT_ID

private static final String DOCUMENT_ID

See Also:

• Constant Field Values

Constructor Details

IncrementalSnapshotIT

public IncrementalSnapshotIT()

Method Details

before

public void before()

after

public void after()

connectorClass

protected Class<io.debezium.connector.mongodb.MongoDbConnector> connectorClass()

config

protected Configuration.Builder config()

dataCollectionName

protected String dataCollectionName()

dataCollectionNames

protected List<String> dataCollectionNames()

fullDataCollectionName

protected String fullDataCollectionName()

fullDataCollectionNames

protected List<String> fullDataCollectionNames()

topicName

protected String topicName()

topicNames

protected List<String> topicNames()

populateDataCollection

protected void populateDataCollection(String dataCollectionName)

populateDataCollection

protected void populateDataCollection()

populateDataCollections

protected void populateDataCollections()

insertAdditionalData

protected void insertAdditionalData()

insertAdditionalData

protected void insertAdditionalData(String collectionName)

updateData

protected void updateData(int batchSize)

startConnector

protected void startConnector(Function<Configuration.Builder,Configuration.Builder> custConfig)

startConnector

protected void startConnector(Function<Configuration.Builder,Configuration.Builder> custConfig,
 io.debezium.engine.DebeziumEngine.CompletionCallback callback)

startConnector

protected void startConnector()

waitForConnectorToStart

protected void waitForConnectorToStart()

sendAdHocSnapshotSignal

protected void sendAdHocSnapshotSignal(String... dataCollectionIds)
                                throws SQLException

Throws:

SQLException

sendAdHocSnapshotStopSignal

protected void sendAdHocSnapshotStopSignal(String... dataCollectionIds)
                                    throws SQLException

Throws:

SQLException

sendAdHocSnapshotSignal

protected void sendAdHocSnapshotSignal()
                                throws SQLException

Throws:

SQLException

sendPauseSignal

protected void sendPauseSignal()
                        throws SQLException

Throws:

SQLException

sendResumeSignal

protected void sendResumeSignal()
                         throws SQLException

Throws:

SQLException

consumeMixedWithIncrementalSnapshot

protected Map<Integer,Integer> consumeMixedWithIncrementalSnapshot(int recordCount)
                                                            throws InterruptedException

Throws:

InterruptedException

consumeMixedWithIncrementalSnapshot

protected Map<Integer,Integer> consumeMixedWithIncrementalSnapshot(int recordCount,
 String topicName)
                                                            throws InterruptedException

Throws:

InterruptedException

extractFieldValue

protected Integer extractFieldValue(org.apache.kafka.connect.source.SourceRecord record)

consumeMixedWithIncrementalSnapshot

protected <V> Map<Integer,V> consumeMixedWithIncrementalSnapshot(int recordCount,
 Function<org.apache.kafka.connect.source.SourceRecord,V> valueConverter,
 Predicate<Map.Entry<Integer,V>> dataCompleted,
 Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer,
 String topicName)
                                                          throws InterruptedException

Throws:

InterruptedException

consumeMixedWithIncrementalSnapshot

protected <V,
K> Map<K,V> consumeMixedWithIncrementalSnapshot(int recordCount,
 Predicate<Map.Entry<K,V>> dataCompleted,
 Function<org.apache.kafka.connect.data.Struct,K> idCalculator,
 Function<org.apache.kafka.connect.source.SourceRecord,V> valueConverter,
 String topicName,
 Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
                                                      throws InterruptedException

Throws:

InterruptedException

consumeRecordsMixedWithIncrementalSnapshot

protected Map<Integer,org.apache.kafka.connect.source.SourceRecord> consumeRecordsMixedWithIncrementalSnapshot(int recordCount)
                                                                                                        throws InterruptedException

Throws:

InterruptedException

consumeMixedWithIncrementalSnapshot

protected Map<Integer,Integer> consumeMixedWithIncrementalSnapshot(int recordCount,
 Predicate<Map.Entry<Integer,Integer>> dataCompleted,
 Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
                                                            throws InterruptedException

Throws:

InterruptedException

consumeRecordsMixedWithIncrementalSnapshot

protected Map<Integer,org.apache.kafka.connect.source.SourceRecord> consumeRecordsMixedWithIncrementalSnapshot(int recordCount,
 Predicate<Map.Entry<Integer,org.apache.kafka.connect.source.SourceRecord>> dataCompleted,
 Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
                                                                                                        throws InterruptedException

Throws:

InterruptedException

valueFieldName

protected String valueFieldName()

pkFieldName

protected String pkFieldName()

snapshotOnly

private <K> void snapshotOnly(K initialId,
 Function<K,K> idGenerator)
                       throws Exception

Throws:

Exception

shouldStreamWithDatabaseIncludeList

public void shouldStreamWithDatabaseIncludeList()
                                         throws InterruptedException

Throws:

InterruptedException

snapshotOnlyInt32

public void snapshotOnlyInt32()
                       throws Exception

Throws:

Exception

snapshotOnlyWithInt64

public void snapshotOnlyWithInt64()
                           throws Exception

Throws:

Exception

snapshotOnlyDouble

public void snapshotOnlyDouble()
                        throws Exception

Throws:

Exception

snapshotOnlyDecimal128

public void snapshotOnlyDecimal128()
                            throws Exception

Throws:

Exception

snapshotOnlyObjectId

public void snapshotOnlyObjectId()
                          throws Exception

Throws:

Exception

snapshotOnlyString

public void snapshotOnlyString()
                        throws Exception

Throws:

Exception

invalidTablesInTheList

public void invalidTablesInTheList()
                            throws Exception

Throws:

Exception

snapshotOnlyWithRestart

public void snapshotOnlyWithRestart()
                             throws Exception

Throws:

Exception

inserts

public void inserts()
             throws Exception

Throws:

Exception

updates

public void updates()
             throws Exception

Throws:

Exception

updatesWithRestart

public void updatesWithRestart()
                        throws Exception

Throws:

Exception

updatesLargeChunk

public void updatesLargeChunk()
                       throws Exception

Throws:

Exception

stopCurrentIncrementalSnapshotWithoutCollectionsAndTakeNewNewIncrementalSnapshotAfterRestart

@FixFor("DBZ-4271")
public void stopCurrentIncrementalSnapshotWithoutCollectionsAndTakeNewNewIncrementalSnapshotAfterRestart()
                                                                                                  throws Exception

Throws:

Exception

stopCurrentIncrementalSnapshotWithAllCollectionsAndTakeNewNewIncrementalSnapshotAfterRestart

@FixFor("DBZ-4271")
public void stopCurrentIncrementalSnapshotWithAllCollectionsAndTakeNewNewIncrementalSnapshotAfterRestart()
                                                                                                  throws Exception

Throws:

Exception

removeNotYetCapturedCollectionFromInProgressIncrementalSnapshot

@FixFor("DBZ-4271")
public void removeNotYetCapturedCollectionFromInProgressIncrementalSnapshot()
                                                                     throws Exception

Throws:

Exception

removeStartedCapturedCollectionFromInProgressIncrementalSnapshot

@FixFor("DBZ-4271")
public void removeStartedCapturedCollectionFromInProgressIncrementalSnapshot()
                                                                      throws Exception

Throws:

Exception

pauseDuringSnapshot

public void pauseDuringSnapshot()
                         throws Exception

Throws:

Exception

insertInsertWatermarkingStrategy

public void insertInsertWatermarkingStrategy()
                                      throws Exception

Throws:

Exception

insertDeleteWatermarkingStrategy

public void insertDeleteWatermarkingStrategy()
                                      throws Exception

Throws:

Exception

assertCloseEventCount

private void assertCloseEventCount(Consumer<Long> consumer)

sendAdHocSnapshotSignalAndWait

protected void sendAdHocSnapshotSignalAndWait(String... collectionIds)
                                       throws Exception

Throws:

Exception

sendAdHocSnapshotStopSignalAndWait

protected void sendAdHocSnapshotStopSignalAndWait(String... collectionIds)
                                           throws Exception

Throws:

Exception

consumeAnyRemainingIncrementalSnapshotEventsAndCheckForStopMessage

protected boolean consumeAnyRemainingIncrementalSnapshotEventsAndCheckForStopMessage(LogInterceptor interceptor,
 String stopMessage)
                                                                              throws Exception

Throws:

Exception

getMaximumEnqueuedRecordCount

protected int getMaximumEnqueuedRecordCount()

Overrides:

getMaximumEnqueuedRecordCount in class AbstractConnectorTest