Package io.debezium.connector.mongodb

Class ShardedIncrementalSnapshotIT

java.lang.Object

io.debezium.embedded.AbstractConnectorTest

io.debezium.connector.mongodb.AbstractShardedMongoConnectorIT

io.debezium.connector.mongodb.ShardedIncrementalSnapshotIT

All Implemented Interfaces:

Testing

public class ShardedIncrementalSnapshotIT
extends AbstractShardedMongoConnectorIT

Nested Class Summary

Nested classes/interfaces inherited from class io.debezium.embedded.AbstractConnectorTest

AbstractConnectorTest.SourceRecords

Nested classes/interfaces inherited from interface io.debezium.util.Testing

Testing.Debug, Testing.Files, Testing.InterruptableFunction, Testing.Network, Testing.Print, Testing.Timer

Field Summary

Fields

Modifier and Type	Field	Description
private static final int	CHUNK_SIZE
private static final String	COLLECTION_NAME
private static final String	DATABASE_NAME
private static final String	DOCUMENT_ID
private static final String	FULL_COLLECTION_NAME
private static final int	MAXIMUM_NO_RECORDS_CONSUMES
protected static final int	ROW_COUNT
private static final String	SIGNAL_COLLECTION_NAME

Fields inherited from class io.debezium.connector.mongodb.AbstractShardedMongoConnectorIT

DEFAULT_COLLECTION, DEFAULT_DATABASE, DEFAULT_SHARDING_KEY, mongo

Fields inherited from class io.debezium.embedded.AbstractConnectorTest

consumedLines, engine, logger, logTestName, OFFSET_STORE_PATH, pollTimeoutInMs, skipTestRule

Constructor Summary

Constructors

Constructor	Description
ShardedIncrementalSnapshotIT()

Method Summary

Modifier and Type	Method	Description
protected Configuration.Builder	config()
protected Class<io.debezium.connector.mongodb.MongoDbConnector>	connectorClass()
protected Map<Integer,Integer>	consumeMixedWithIncrementalSnapshot(int recordCount, String topicName)
protected <V> Map<Integer,V>	consumeMixedWithIncrementalSnapshot(int recordCount, Function<org.apache.kafka.connect.source.SourceRecord,V> valueConverter, Predicate<Map.Entry<Integer,V>> dataCompleted, Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer, String topicName)
protected <V, K> Map<K,V>	consumeMixedWithIncrementalSnapshot(int recordCount, Predicate<Map.Entry<K,V>> dataCompleted, Function<org.apache.kafka.connect.data.Struct,K> idCalculator, Function<org.apache.kafka.connect.source.SourceRecord,V> valueConverter, String topicName, Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
protected Integer	extractFieldValue(org.apache.kafka.connect.source.SourceRecord record)
protected String	fullDataCollectionName()
protected String	pkFieldName()
protected void	sendAdHocSnapshotSignal()
protected void	sendAdHocSnapshotSignal(String... dataCollectionIds)
protected Map<String,String>	shardedCollections()
protected String	shardedDatabase()
private <K> void	snapshotOnly(K initialId, Function<K,K> idGenerator)
void	snapshotOnlyWithInt64()
protected void	startConnector()
protected void	startConnector(Function<Configuration.Builder,Configuration.Builder> custConfig, io.debezium.engine.DebeziumEngine.CompletionCallback callback)
protected String	topicName()
protected String	valueFieldName()
protected void	waitForConnectorToStart()

Methods inherited from class io.debezium.connector.mongodb.AbstractShardedMongoConnectorIT

afterAll, afterEach, beforeAll, beforeEach, connect, insertDocuments, insertDocumentsInTx, shardedCollection

Methods inherited from class io.debezium.embedded.AbstractConnectorTest

assertBeginTransaction, assertConfigurationErrors, assertConfigurationErrors, assertConfigurationErrors, assertConnectorIsRunning, assertConnectorNotRunning, assertDelete, assertEndTransaction, assertEngineIsRunning, assertHasNoSourceQuery, assertInsert, assertKey, assertNoConfigurationErrors, assertNoRecordsToConsume, assertOffset, assertOffset, assertOnlyTransactionRecordsToConsume, assertRecordTransactionMetadata, assertSchemaMatchesStruct, assertSchemaMatchesStruct, assertSourceQuery, assertTombstone, assertTombstone, assertUpdate, assertValueField, configValue, consumeAvailableRecords, consumeDmlRecordsByTopic, consumeDmlRecordsByTopic, consumeDmlRecordsByTopic, consumeRecord, consumeRecords, consumeRecords, consumeRecords, consumeRecordsButSkipUntil, consumeRecordsByTopic, consumeRecordsByTopic, consumeRecordsByTopic, consumeRecordsByTopicUntil, consumeRecordsUntil, debug, getConsumer, getMaximumEnqueuedRecordCount, getSnapshotMetricsObjectName, getSnapshotMetricsObjectName, getStreamingMetricsObjectName, getStreamingMetricsObjectName, getStreamingNamespace, initializeConnectorTestFramework, isStreamingRunning, isStreamingRunning, isTransactionRecord, loggingCompletion, print, readLastCommittedOffset, readLastCommittedOffsets, setConsumeTimeout, skipAvroValidation, start, start, start, start, start, start, start, startAndConsumeTillEnd, startAndConsumeTillEnd, stopConnector, stopConnector, validate, waitForAvailableRecords, waitForConnectorShutdown, waitForNotInitialState, waitForSnapshotToBeCompleted, waitForSnapshotToBeCompleted, waitForStreamingRunning, waitForStreamingRunning, waitTimeForRecords, waitTimeForRecordsAfterNulls

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface io.debezium.util.Testing

once, once, resetBeforeEachTest, time, time

Field Details

ROW_COUNT

protected static final int ROW_COUNT

See Also:

• Constant Field Values

MAXIMUM_NO_RECORDS_CONSUMES

private static final int MAXIMUM_NO_RECORDS_CONSUMES

See Also:

• Constant Field Values

DATABASE_NAME

private static final String DATABASE_NAME

See Also:

• Constant Field Values

COLLECTION_NAME

private static final String COLLECTION_NAME

See Also:

• Constant Field Values

SIGNAL_COLLECTION_NAME

private static final String SIGNAL_COLLECTION_NAME

See Also:

• Constant Field Values

FULL_COLLECTION_NAME

private static final String FULL_COLLECTION_NAME

See Also:

• Constant Field Values

DOCUMENT_ID

private static final String DOCUMENT_ID

See Also:

• Constant Field Values

CHUNK_SIZE

private static final int CHUNK_SIZE

See Also:

• Constant Field Values

Constructor Details

ShardedIncrementalSnapshotIT

public ShardedIncrementalSnapshotIT()

Method Details

shardedDatabase

protected String shardedDatabase()

Overrides:

shardedDatabase in class AbstractShardedMongoConnectorIT

shardedCollections

protected Map<String,String> shardedCollections()

Overrides:

shardedCollections in class AbstractShardedMongoConnectorIT

fullDataCollectionName

protected String fullDataCollectionName()

topicName

protected String topicName()

config

protected Configuration.Builder config()

snapshotOnlyWithInt64

public void snapshotOnlyWithInt64()
                           throws Exception

Throws:

Exception

snapshotOnly

private <K> void snapshotOnly(K initialId,
 Function<K,K> idGenerator)
                       throws Exception

Throws:

Exception

valueFieldName

protected String valueFieldName()

pkFieldName

protected String pkFieldName()

connectorClass

protected Class<io.debezium.connector.mongodb.MongoDbConnector> connectorClass()

startConnector

protected void startConnector(Function<Configuration.Builder,Configuration.Builder> custConfig,
 io.debezium.engine.DebeziumEngine.CompletionCallback callback)

startConnector

protected void startConnector()

waitForConnectorToStart

protected void waitForConnectorToStart()

sendAdHocSnapshotSignal

protected void sendAdHocSnapshotSignal()
                                throws SQLException

Throws:

SQLException

sendAdHocSnapshotSignal

protected void sendAdHocSnapshotSignal(String... dataCollectionIds)
                                throws SQLException

Throws:

SQLException

consumeMixedWithIncrementalSnapshot

protected Map<Integer,Integer> consumeMixedWithIncrementalSnapshot(int recordCount,
 String topicName)
                                                            throws InterruptedException

Throws:

InterruptedException

extractFieldValue

protected Integer extractFieldValue(org.apache.kafka.connect.source.SourceRecord record)

consumeMixedWithIncrementalSnapshot

protected <V> Map<Integer,V> consumeMixedWithIncrementalSnapshot(int recordCount,
 Function<org.apache.kafka.connect.source.SourceRecord,V> valueConverter,
 Predicate<Map.Entry<Integer,V>> dataCompleted,
 Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer,
 String topicName)
                                                          throws InterruptedException

Throws:

InterruptedException

consumeMixedWithIncrementalSnapshot

protected <V,
K> Map<K,V> consumeMixedWithIncrementalSnapshot(int recordCount,
 Predicate<Map.Entry<K,V>> dataCompleted,
 Function<org.apache.kafka.connect.data.Struct,K> idCalculator,
 Function<org.apache.kafka.connect.source.SourceRecord,V> valueConverter,
 String topicName,
 Consumer<List<org.apache.kafka.connect.source.SourceRecord>> recordConsumer)
                                                      throws InterruptedException

Throws:

InterruptedException