Package io.debezium.connector.mongodb.snapshot

Class MongoDbIncrementalSnapshotChangeEventSource

java.lang.Object

io.debezium.connector.mongodb.snapshot.MongoDbIncrementalSnapshotChangeEventSource

All Implemented Interfaces:

IncrementalSnapshotChangeEventSource<MongoDbPartition,CollectionId>

@NotThreadSafe
public class MongoDbIncrementalSnapshotChangeEventSource
extends Object
implements IncrementalSnapshotChangeEventSource<MongoDbPartition,CollectionId>

An incremental snapshot change event source that emits events from a MongoDB change stream interleaved with snapshot events.

Field Summary

Fields

Modifier and Type	Field	Description
private final Clock	clock
private final MongoDbSchema	collectionSchema
private final MongoDbConnectorConfig	connectorConfig
protected IncrementalSnapshotContext<CollectionId>	context
private MongoDbCollectionSchema	currentCollection
private final DataChangeEventListener<MongoDbPartition>	dataListener
protected EventDispatcher<MongoDbPartition,CollectionId>	dispatcher
private static final String	DOCUMENT_ID
private final ExecutorService	incrementalSnapshotThreadPool
private static final org.slf4j.Logger	LOGGER
private MongoDbConnection	mongo
protected final NotificationService<MongoDbPartition,? extends OffsetContext>	notificationService
private final SnapshotProgressListener<MongoDbPartition>	progressListener
private final CollectionId	signallingCollectionId
private final MongoDbTaskContext	taskContext
private long	totalRowsScanned
protected final Map<org.apache.kafka.connect.data.Struct,Object[]>	window

Constructor Summary

Constructors

Constructor

Description

MongoDbIncrementalSnapshotChangeEventSource(MongoDbConnectorConfig config, MongoDbTaskContext taskContext, EventDispatcher<MongoDbPartition,CollectionId> dispatcher, MongoDbSchema collectionSchema, Clock clock, SnapshotProgressListener<MongoDbPartition> progressListener, DataChangeEventListener<MongoDbPartition> dataChangeEventListener, NotificationService<MongoDbPartition,? extends OffsetContext> notificationService)

Method Summary

Modifier and Type	Method	Description
protected Object[]	addChunkToExecutor(com.mongodb.client.MongoCollection<org.bson.BsonDocument> collection, Object[] lastRow, List<Future<?>> futureList, Object[] lastChunkKey)
void	addDataCollectionNamesToSnapshot(SignalPayload<MongoDbPartition> signalPayload, SnapshotConfiguration snapshotConfiguration)
void	closeWindow(MongoDbPartition partition, String id, OffsetContext offsetContext)
private void	collectionScanCompleted(MongoDbPartition partition)
private org.bson.Document	constructQueryPredicate(Object[] startKey, Object[] endKey, String additionalConditions)
private void	createDataEventsForDataCollection(MongoDbPartition partition)	Dispatches the data change events for the records of a single table.
protected void	deduplicateWindow(io.debezium.spi.schema.DataCollectionId dataCollectionId, Object key)
protected void	emitWindowClose(Partition partition, OffsetContext offsetContext)	Update high watermark for the incremental snapshot chunk
protected void	emitWindowOpen()	Update low watermark for the incremental snapshot chunk
private List<String>	expandAndDedupeDataCollectionIds(List<String> collectionIds)	Expands the string-based list of data collection ids if supplied using regex to a list of all matching explicit data collection ids.
private String	getAdditionalConditions()
protected ChangeRecordEmitter<MongoDbPartition>	getChangeRecordEmitter(MongoDbPartition partition, OffsetContext offsetContext, Object[] row)	Returns a ChangeRecordEmitter producing the change records for the given table row.
private static MongoDbOffsetContext	getMongoDbOffsetContext(OffsetContext offsetContext)
private String	getStripedAdditionalConditions(String additionalConditions)
private Threads.Timer	getTableScanLogTimer()
private WatermarkWindowCloser	getWatermarkWindowCloser(CommonConnectorConfig connectorConfig, MongoDbConnection mongoDbConnection, CollectionId collectionId)
private void	incrementTableRowsScanned(long rows)
void	init(MongoDbPartition partition, OffsetContext offsetContext)
private Object[]	keyFromRow(Object[] row)
private void	nextDataCollection(MongoDbPartition partition, OffsetContext offsetContext)
void	pauseSnapshot(MongoDbPartition partition, OffsetContext offsetContext)
protected void	postIncrementalSnapshotCompleted()
protected void	postReadChunk(IncrementalSnapshotContext<CollectionId> context)
protected void	preReadChunk(IncrementalSnapshotContext<CollectionId> context)
void	processMessage(MongoDbPartition partition, io.debezium.spi.schema.DataCollectionId dataCollectionId, Object key, OffsetContext offsetContext)
private void	queryChunk(com.mongodb.client.MongoCollection<org.bson.BsonDocument> collection, Object[] startKey, Object[] endKey)
protected void	readChunk(MongoDbPartition partition, OffsetContext offsetContext)
private Object[]	readMaximumKey()
void	resumeSnapshot(MongoDbPartition partition, OffsetContext offsetContext)
protected void	sendEvent(EventDispatcher<MongoDbPartition,CollectionId> dispatcher, MongoDbPartition partition, MongoDbOffsetContext offsetContext, Object[] row)
protected void	sendWindowEvents(MongoDbPartition partition, MongoDbOffsetContext offsetContext)
protected void	setContext(IncrementalSnapshotContext<CollectionId> context)
void	stopSnapshot(MongoDbPartition partition, OffsetContext offsetContext, Map<String,Object> additionalData, List<String> dataCollectionPatterns)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface io.debezium.pipeline.source.snapshot.incremental.IncrementalSnapshotChangeEventSource

processFilteredEvent, processHeartbeat, processSchemaChange, processTransactionCommittedEvent, processTransactionStartedEvent

Field Details

DOCUMENT_ID

private static final String DOCUMENT_ID

See Also:

• Constant Field Values

LOGGER

private static final org.slf4j.Logger LOGGER

connectorConfig

private final MongoDbConnectorConfig connectorConfig

taskContext

private final MongoDbTaskContext taskContext

clock

private final Clock clock

collectionSchema

private final MongoDbSchema collectionSchema

progressListener

private final SnapshotProgressListener<MongoDbPartition> progressListener

dataListener

private final DataChangeEventListener<MongoDbPartition> dataListener

totalRowsScanned

private long totalRowsScanned

currentCollection

private MongoDbCollectionSchema currentCollection

dispatcher

protected EventDispatcher<MongoDbPartition,CollectionId> dispatcher

context

protected IncrementalSnapshotContext<CollectionId> context

window

protected final Map<org.apache.kafka.connect.data.Struct,Object[]> window

mongo

private MongoDbConnection mongo

signallingCollectionId

private final CollectionId signallingCollectionId

notificationService

protected final NotificationService<MongoDbPartition,? extends OffsetContext> notificationService

incrementalSnapshotThreadPool

private final ExecutorService incrementalSnapshotThreadPool

Constructor Details

MongoDbIncrementalSnapshotChangeEventSource

public MongoDbIncrementalSnapshotChangeEventSource(MongoDbConnectorConfig config,
 MongoDbTaskContext taskContext,
 EventDispatcher<MongoDbPartition,CollectionId> dispatcher,
 MongoDbSchema collectionSchema,
 Clock clock,
 SnapshotProgressListener<MongoDbPartition> progressListener,
 DataChangeEventListener<MongoDbPartition> dataChangeEventListener,
 NotificationService<MongoDbPartition,? extends OffsetContext> notificationService)

Method Details

closeWindow

public void closeWindow(MongoDbPartition partition,
 String id,
 OffsetContext offsetContext)
                 throws InterruptedException

Specified by:

closeWindow in interface IncrementalSnapshotChangeEventSource<MongoDbPartition,CollectionId>

Throws:

InterruptedException

pauseSnapshot

public void pauseSnapshot(MongoDbPartition partition,
 OffsetContext offsetContext)
                   throws InterruptedException

Specified by:

pauseSnapshot in interface IncrementalSnapshotChangeEventSource<MongoDbPartition,CollectionId>

Throws:

InterruptedException

resumeSnapshot

public void resumeSnapshot(MongoDbPartition partition,
 OffsetContext offsetContext)
                    throws InterruptedException

Specified by:

resumeSnapshot in interface IncrementalSnapshotChangeEventSource<MongoDbPartition,CollectionId>

Throws:

InterruptedException

sendWindowEvents

protected void sendWindowEvents(MongoDbPartition partition,
 MongoDbOffsetContext offsetContext)
                         throws InterruptedException

Throws:

InterruptedException

sendEvent

protected void sendEvent(EventDispatcher<MongoDbPartition,CollectionId> dispatcher,
 MongoDbPartition partition,
 MongoDbOffsetContext offsetContext,
 Object[] row)
                  throws InterruptedException

Throws:

InterruptedException

getMongoDbOffsetContext

private static MongoDbOffsetContext getMongoDbOffsetContext(OffsetContext offsetContext)

getChangeRecordEmitter

protected ChangeRecordEmitter<MongoDbPartition> getChangeRecordEmitter(MongoDbPartition partition,
 OffsetContext offsetContext,
 Object[] row)

Returns a ChangeRecordEmitter producing the change records for the given table row.

deduplicateWindow

protected void deduplicateWindow(io.debezium.spi.schema.DataCollectionId dataCollectionId,
 Object key)

emitWindowOpen

protected void emitWindowOpen()
                       throws InterruptedException

Update low watermark for the incremental snapshot chunk

Throws:

InterruptedException

emitWindowClose

protected void emitWindowClose(Partition partition,
 OffsetContext offsetContext)
                        throws Exception

Update high watermark for the incremental snapshot chunk

Throws:

Exception

getWatermarkWindowCloser

private WatermarkWindowCloser getWatermarkWindowCloser(CommonConnectorConfig connectorConfig,
 MongoDbConnection mongoDbConnection,
 CollectionId collectionId)

init

public void init(MongoDbPartition partition,
 OffsetContext offsetContext)

Specified by:

init in interface IncrementalSnapshotChangeEventSource<MongoDbPartition,CollectionId>

readChunk

protected void readChunk(MongoDbPartition partition,
 OffsetContext offsetContext)
                  throws InterruptedException

Throws:

InterruptedException

nextDataCollection

private void nextDataCollection(MongoDbPartition partition,
 OffsetContext offsetContext)

readMaximumKey

private Object[] readMaximumKey()
                         throws InterruptedException

Throws:

InterruptedException

addDataCollectionNamesToSnapshot

public void addDataCollectionNamesToSnapshot(SignalPayload<MongoDbPartition> signalPayload,
 SnapshotConfiguration snapshotConfiguration)
                                      throws InterruptedException

Specified by:

addDataCollectionNamesToSnapshot in interface IncrementalSnapshotChangeEventSource<MongoDbPartition,CollectionId>

Throws:

InterruptedException

stopSnapshot

public void stopSnapshot(MongoDbPartition partition,
 OffsetContext offsetContext,
 Map<String,Object> additionalData,
 List<String> dataCollectionPatterns)

Specified by:

stopSnapshot in interface IncrementalSnapshotChangeEventSource<MongoDbPartition,CollectionId>

expandAndDedupeDataCollectionIds

private List<String> expandAndDedupeDataCollectionIds(List<String> collectionIds)

Expands the string-based list of data collection ids if supplied using regex to a list of all matching explicit data collection ids.

createDataEventsForDataCollection

private void createDataEventsForDataCollection(MongoDbPartition partition)
                                        throws InterruptedException

Dispatches the data change events for the records of a single table.

Throws:

InterruptedException

addChunkToExecutor

protected Object[] addChunkToExecutor(com.mongodb.client.MongoCollection<org.bson.BsonDocument> collection,
 Object[] lastRow,
 List<Future<?>> futureList,
 Object[] lastChunkKey)

queryChunk

private void queryChunk(com.mongodb.client.MongoCollection<org.bson.BsonDocument> collection,
 Object[] startKey,
 Object[] endKey)

constructQueryPredicate

private org.bson.Document constructQueryPredicate(Object[] startKey,
 Object[] endKey,
 String additionalConditions)

getAdditionalConditions

private String getAdditionalConditions()

getStripedAdditionalConditions

private String getStripedAdditionalConditions(String additionalConditions)

incrementTableRowsScanned

private void incrementTableRowsScanned(long rows)

collectionScanCompleted

private void collectionScanCompleted(MongoDbPartition partition)

getTableScanLogTimer

private Threads.Timer getTableScanLogTimer()

keyFromRow

private Object[] keyFromRow(Object[] row)

setContext

protected void setContext(IncrementalSnapshotContext<CollectionId> context)

preReadChunk

protected void preReadChunk(IncrementalSnapshotContext<CollectionId> context)

postReadChunk

protected void postReadChunk(IncrementalSnapshotContext<CollectionId> context)

postIncrementalSnapshotCompleted

protected void postIncrementalSnapshotCompleted()

processMessage

public void processMessage(MongoDbPartition partition,
 io.debezium.spi.schema.DataCollectionId dataCollectionId,
 Object key,
 OffsetContext offsetContext)
                    throws InterruptedException

Specified by:

processMessage in interface IncrementalSnapshotChangeEventSource<MongoDbPartition,CollectionId>

Throws:

InterruptedException