Package io.debezium.connector.mongodb

Class IncrementalSnapshotIT

java.lang.Object

io.debezium.embedded.AbstractConnectorTest

io.debezium.connector.mongodb.AbstractMongoConnectorIT

io.debezium.connector.mongodb.IncrementalSnapshotIT

All Implemented Interfaces:

Testing

public class IncrementalSnapshotIT
extends AbstractMongoConnectorIT

Test to verify incremental snapshotting for MongoDB.

Author:

Jiri Pechanec

Nested Class Summary

Nested classes/interfaces inherited from class io.debezium.embedded.AbstractConnectorTest

AbstractConnectorTest.SourceRecords

Nested classes/interfaces inherited from interface io.debezium.util.Testing

Testing.Debug, Testing.Files, Testing.InterruptableFunction, Testing.Network, Testing.Print, Testing.Timer

Field Summary

Fields

Modifier and Type	Field	Description
private static final String	COLLECTION_NAME
private static final String	DATABASE_NAME
protected static final Path	DB_HISTORY_PATH
private static final String	DOCUMENT_ID
private static final String	FULL_COLLECTION_NAME
private static final int	MAXIMUM_NO_RECORDS_CONSUMES
protected static final int	ROW_COUNT
org.junit.rules.TestRule	skipForOplog

Fields inherited from class io.debezium.connector.mongodb.AbstractMongoConnectorIT

config, context, logInterceptor

Fields inherited from class io.debezium.embedded.AbstractConnectorTest

engine, logger, logTestName, OFFSET_STORE_PATH, pollTimeoutInMs, skipTestRule

Constructor Summary

Constructors

Constructor	Description
IncrementalSnapshotIT()

Method Summary

Modifier and Type	Method	Description
void	after()
void	before()
protected Configuration.Builder	config()
protected Class<io.debezium.connector.mongodb.MongoDbConnector>	connectorClass()
protected Map<Integer,Integer>	consumeMixedWithIncrementalSnapshot(int recordCount)
protected <V> Map<Integer,V>	consumeMixedWithIncrementalSnapshot(int recordCount, Function<org.apache.kafka.connect.source.SourceRecord,V> valueConverter, Predicate<Map.Entry<Integer,V>> dataCompleted, Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
protected Map<Integer,Integer>	consumeMixedWithIncrementalSnapshot(int recordCount, Predicate<Map.Entry<Integer,Integer>> dataCompleted, Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
protected <V> Map<Integer,V>	consumeMixedWithIncrementalSnapshot(int recordCount, Predicate<Map.Entry<Integer,V>> dataCompleted, Function<org.apache.kafka.connect.data.Struct,Integer> idCalculator, Function<org.apache.kafka.connect.source.SourceRecord,V> valueConverter, String topicName, Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
protected Map<Integer,org.apache.kafka.connect.source.SourceRecord>	consumeRecordsMixedWithIncrementalSnapshot(int recordCount)
protected Map<Integer,org.apache.kafka.connect.source.SourceRecord>	consumeRecordsMixedWithIncrementalSnapshot(int recordCount, Predicate<Map.Entry<Integer,org.apache.kafka.connect.source.SourceRecord>> dataCompleted, Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
protected String	dataCollectionName()
protected Integer	extractFieldValue(org.apache.kafka.connect.source.SourceRecord record)
protected String	fullDataCollectionName()
protected int	getMaximumEnqueuedRecordCount()
protected void	insertAdditionalData()
void	inserts()
void	invalidTablesInTheList()
protected String	pkFieldName()
protected void	populateDataCollection()
protected void	populateDataCollection(io.debezium.connector.mongodb.ConnectionContext.MongoPrimary connection)
protected void	populateDataCollection(io.debezium.connector.mongodb.ConnectionContext.MongoPrimary connection, String dataCollectionName)
protected void	sendAdHocSnapshotSignal()
protected void	sendAdHocSnapshotSignal(String... dataCollectionIds)
void	snapshotOnly()
void	snapshotOnlyWithRestart()
protected void	startConnector()
protected void	startConnector(io.debezium.engine.DebeziumEngine.CompletionCallback callback)
protected void	startConnector(Function<Configuration.Builder,Configuration.Builder> custConfig)
protected void	startConnector(Function<Configuration.Builder,Configuration.Builder> custConfig, io.debezium.engine.DebeziumEngine.CompletionCallback callback)
protected String	topicName()
protected void	updateData(int batchSize)
void	updates()
void	updatesLargeChunk()
void	updatesWithRestart()
protected String	valueFieldName()
protected void	waitForConnectorToStart()

Methods inherited from class io.debezium.connector.mongodb.AbstractMongoConnectorIT

afterEach, beforEach, connectionErrorHandler, deleteDocuments, dropAndInsertDocuments, insertDocuments, insertDocumentsInTx, loadTestDocuments, primary, storeDocuments, storeDocuments, updateDocument, updateDocumentsInTx

Methods inherited from class io.debezium.embedded.AbstractConnectorTest

assertBeginTransaction, assertConfigurationErrors, assertConfigurationErrors, assertConfigurationErrors, assertConnectorIsRunning, assertConnectorNotRunning, assertDelete, assertEndTransaction, assertEngineIsRunning, assertHasNoSourceQuery, assertInsert, assertKey, assertNoConfigurationErrors, assertNoRecordsToConsume, assertOffset, assertOffset, assertOnlyTransactionRecordsToConsume, assertRecordTransactionMetadata, assertSchemaMatchesStruct, assertSchemaMatchesStruct, assertSourceQuery, assertTombstone, assertTombstone, assertUpdate, assertValueField, configValue, consumeAvailableRecords, consumeDmlRecordsByTopic, consumeDmlRecordsByTopic, consumeDmlRecordsByTopic, consumeRecord, consumeRecords, consumeRecords, consumeRecords, consumeRecordsByTopic, consumeRecordsByTopic, consumeRecordsByTopic, consumeRecordsByTopicUntil, consumeRecordsUntil, debug, getSnapshotMetricsObjectName, getStreamingMetricsObjectName, getStreamingMetricsObjectName, getStreamingNamespace, initializeConnectorTestFramework, isStreamingRunning, isStreamingRunning, isTransactionRecord, loggingCompletion, print, readLastCommittedOffset, readLastCommittedOffsets, setConsumeTimeout, skipAvroValidation, start, start, start, start, start, startAndConsumeTillEnd, startAndConsumeTillEnd, stopConnector, stopConnector, validate, waitForAvailableRecords, waitForConnectorShutdown, waitForSnapshotToBeCompleted, waitForStreamingRunning, waitForStreamingRunning, waitTimeForRecords, waitTimeForRecordsAfterNulls

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface io.debezium.util.Testing

once, once, resetBeforeEachTest, time, time

Field Details

ROW_COUNT

protected static final int ROW_COUNT

See Also:

• Constant Field Values

MAXIMUM_NO_RECORDS_CONSUMES

private static final int MAXIMUM_NO_RECORDS_CONSUMES

See Also:

• Constant Field Values

DATABASE_NAME

private static final String DATABASE_NAME

See Also:

• Constant Field Values

COLLECTION_NAME

private static final String COLLECTION_NAME

See Also:

• Constant Field Values

FULL_COLLECTION_NAME

private static final String FULL_COLLECTION_NAME

See Also:

• Constant Field Values

DOCUMENT_ID

private static final String DOCUMENT_ID

See Also:

• Constant Field Values

DB_HISTORY_PATH

protected static final Path DB_HISTORY_PATH

skipForOplog

public org.junit.rules.TestRule skipForOplog

Constructor Details

IncrementalSnapshotIT

public IncrementalSnapshotIT()

Method Details

before

public void before()

after

public void after()

connectorClass

protected Class<io.debezium.connector.mongodb.MongoDbConnector> connectorClass()

config

protected Configuration.Builder config()

dataCollectionName

protected String dataCollectionName()

fullDataCollectionName

protected String fullDataCollectionName()

topicName

protected String topicName()

populateDataCollection

protected void populateDataCollection(io.debezium.connector.mongodb.ConnectionContext.MongoPrimary connection,
 String dataCollectionName)

populateDataCollection

protected void populateDataCollection(io.debezium.connector.mongodb.ConnectionContext.MongoPrimary connection)

populateDataCollection

protected void populateDataCollection()

insertAdditionalData

protected void insertAdditionalData()

updateData

protected void updateData(int batchSize)

startConnector

protected void startConnector(io.debezium.engine.DebeziumEngine.CompletionCallback callback)

startConnector

protected void startConnector(Function<Configuration.Builder,Configuration.Builder> custConfig)

startConnector

protected void startConnector(Function<Configuration.Builder,Configuration.Builder> custConfig,
 io.debezium.engine.DebeziumEngine.CompletionCallback callback)

startConnector

protected void startConnector()

waitForConnectorToStart

protected void waitForConnectorToStart()

sendAdHocSnapshotSignal

protected void sendAdHocSnapshotSignal(String... dataCollectionIds)
                                throws SQLException

Throws:

SQLException

sendAdHocSnapshotSignal

protected void sendAdHocSnapshotSignal()
                                throws SQLException

Throws:

SQLException

consumeMixedWithIncrementalSnapshot

protected Map<Integer,Integer> consumeMixedWithIncrementalSnapshot(int recordCount)
                                                            throws InterruptedException

Throws:

InterruptedException

extractFieldValue

protected Integer extractFieldValue(org.apache.kafka.connect.source.SourceRecord record)

consumeMixedWithIncrementalSnapshot

protected <V> Map<Integer,V> consumeMixedWithIncrementalSnapshot(int recordCount,
 Function<org.apache.kafka.connect.source.SourceRecord,V> valueConverter,
 Predicate<Map.Entry<Integer,V>> dataCompleted,
 Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
                                                          throws InterruptedException

Throws:

InterruptedException

consumeMixedWithIncrementalSnapshot

protected <V> Map<Integer,V> consumeMixedWithIncrementalSnapshot(int recordCount,
 Predicate<Map.Entry<Integer,V>> dataCompleted,
 Function<org.apache.kafka.connect.data.Struct,Integer> idCalculator,
 Function<org.apache.kafka.connect.source.SourceRecord,V> valueConverter,
 String topicName,
 Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
                                                          throws InterruptedException

Throws:

InterruptedException

consumeRecordsMixedWithIncrementalSnapshot

protected Map<Integer,org.apache.kafka.connect.source.SourceRecord> consumeRecordsMixedWithIncrementalSnapshot(int recordCount)
                                                                                                        throws InterruptedException

Throws:

InterruptedException

consumeMixedWithIncrementalSnapshot

protected Map<Integer,Integer> consumeMixedWithIncrementalSnapshot(int recordCount,
 Predicate<Map.Entry<Integer,Integer>> dataCompleted,
 Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
                                                            throws InterruptedException

Throws:

InterruptedException

consumeRecordsMixedWithIncrementalSnapshot

protected Map<Integer,org.apache.kafka.connect.source.SourceRecord> consumeRecordsMixedWithIncrementalSnapshot(int recordCount,
 Predicate<Map.Entry<Integer,org.apache.kafka.connect.source.SourceRecord>> dataCompleted,
 Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
                                                                                                        throws InterruptedException

Throws:

InterruptedException

valueFieldName

protected String valueFieldName()

pkFieldName

protected String pkFieldName()

snapshotOnly

public void snapshotOnly()
                  throws Exception

Throws:

Exception

invalidTablesInTheList

public void invalidTablesInTheList()
                            throws Exception

Throws:

Exception

snapshotOnlyWithRestart

public void snapshotOnlyWithRestart()
                             throws Exception

Throws:

Exception

inserts

public void inserts()
             throws Exception

Throws:

Exception

updates

public void updates()
             throws Exception

Throws:

Exception

updatesWithRestart

public void updatesWithRestart()
                        throws Exception

Throws:

Exception

updatesLargeChunk

public void updatesLargeChunk()
                       throws Exception

Throws:

Exception

getMaximumEnqueuedRecordCount

protected int getMaximumEnqueuedRecordCount()

Overrides:

getMaximumEnqueuedRecordCount in class AbstractConnectorTest