Package io.debezium.connector.mongodb

Class MongoDbIncrementalSnapshotChangeEventSource

java.lang.Object

io.debezium.connector.mongodb.MongoDbIncrementalSnapshotChangeEventSource

All Implemented Interfaces:

IncrementalSnapshotChangeEventSource<MongoDbPartition,CollectionId>

@NotThreadSafe
public class MongoDbIncrementalSnapshotChangeEventSource
extends Object
implements IncrementalSnapshotChangeEventSource<MongoDbPartition,CollectionId>

An incremental snapshot change event source that emits events from a MongoDB change stream interleaved with snapshot events.

Field Summary

Fields

Modifier and Type	Field	Description
private final Clock	clock
private final MongoDbSchema	collectionSchema
private final ConnectionContext	connectionContext
private final MongoDbConnection.ChangeEventSourceConnectionFactory	connections
private final MongoDbConnectorConfig	connectorConfig
protected IncrementalSnapshotContext<CollectionId>	context
private MongoDbCollectionSchema	currentCollection
private final DataChangeEventListener<MongoDbPartition>	dataListener
protected EventDispatcher<MongoDbPartition,CollectionId>	dispatcher
private static final String	DOCUMENT_ID
private static final org.slf4j.Logger	LOGGER
private MongoDbConnection	mongo
private final SnapshotProgressListener<MongoDbPartition>	progressListener
private final ReplicaSets	replicaSets
private CollectionId	signallingCollectionId
private final MongoDbTaskContext	taskContext
private long	totalRowsScanned
protected final Map<org.apache.kafka.connect.data.Struct,Object[]>	window

Constructor Summary

Constructors

Constructor

Description

MongoDbIncrementalSnapshotChangeEventSource(MongoDbConnectorConfig config, MongoDbTaskContext taskContext, MongoDbConnection.ChangeEventSourceConnectionFactory connections, ReplicaSets replicaSets, EventDispatcher<MongoDbPartition,CollectionId> dispatcher, MongoDbSchema collectionSchema, Clock clock, SnapshotProgressListener<MongoDbPartition> progressListener, DataChangeEventListener<MongoDbPartition> dataChangeEventListener)

Method Summary

Modifier and Type	Method	Description
void	addDataCollectionNamesToSnapshot(MongoDbPartition partition, List<String> dataCollectionIds, Optional<String> additionalCondition, Optional<String> surrogateKey, OffsetContext offsetContext)
void	closeWindow(MongoDbPartition partition, String id, OffsetContext offsetContext)
private void	collectionScanCompleted(MongoDbPartition partition)
private void	createDataEventsForDataCollection(MongoDbPartition partition)	Dispatches the data change events for the records of a single table.
protected void	deduplicateWindow(io.debezium.spi.schema.DataCollectionId dataCollectionId, Object key)
protected void	emitWindowClose()	Update high watermark for the incremental snapshot chunk
protected void	emitWindowOpen()	Update low watermark for the incremental snapshot chunk
protected ChangeRecordEmitter<MongoDbPartition>	getChangeRecordEmitter(MongoDbPartition partition, OffsetContext offsetContext, Object[] row)	Returns a ChangeRecordEmitter producing the change records for the given table row.
protected String	getSignalCollectionName(String dataCollectionId)
private Threads.Timer	getTableScanLogTimer()
private void	incrementTableRowsScanned(long rows)
void	init(MongoDbPartition partition, OffsetContext offsetContext)
private Object[]	keyFromRow(Object[] row)
private void	nextDataCollection(MongoDbPartition partition)
void	pauseSnapshot(MongoDbPartition partition, OffsetContext offsetContext)
protected void	postIncrementalSnapshotCompleted()
protected void	postReadChunk(IncrementalSnapshotContext<CollectionId> context)
protected void	preReadChunk(IncrementalSnapshotContext<CollectionId> context)
void	processMessage(MongoDbPartition partition, io.debezium.spi.schema.DataCollectionId dataCollectionId, Object key, OffsetContext offsetContext)
protected void	readChunk(MongoDbPartition partition)
private Object[]	readMaximumKey()
void	resumeSnapshot(MongoDbPartition partition, OffsetContext offsetContext)
protected void	sendEvent(MongoDbPartition partition, EventDispatcher<MongoDbPartition,CollectionId> dispatcher, OffsetContext offsetContext, Object[] row)
protected void	sendWindowEvents(MongoDbPartition partition, OffsetContext offsetContext)
protected void	setContext(IncrementalSnapshotContext<CollectionId> context)
void	stopSnapshot(MongoDbPartition partition, List<String> dataCollectionIds, OffsetContext offsetContext)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface io.debezium.pipeline.source.snapshot.incremental.IncrementalSnapshotChangeEventSource

processFilteredEvent, processHeartbeat, processSchemaChange, processTransactionCommittedEvent, processTransactionStartedEvent

Field Details

DOCUMENT_ID

private static final String DOCUMENT_ID

See Also:

• Constant Field Values

LOGGER

private static final org.slf4j.Logger LOGGER

connectorConfig

private final MongoDbConnectorConfig connectorConfig

clock

private final Clock clock

collectionSchema

private final MongoDbSchema collectionSchema

progressListener

private final SnapshotProgressListener<MongoDbPartition> progressListener

dataListener

private final DataChangeEventListener<MongoDbPartition> dataListener

totalRowsScanned

private long totalRowsScanned

replicaSets

private final ReplicaSets replicaSets

connectionContext

private final ConnectionContext connectionContext

taskContext

private final MongoDbTaskContext taskContext

connections

private final MongoDbConnection.ChangeEventSourceConnectionFactory connections

currentCollection

private MongoDbCollectionSchema currentCollection

dispatcher

protected EventDispatcher<MongoDbPartition,CollectionId> dispatcher

context

protected IncrementalSnapshotContext<CollectionId> context

window

protected final Map<org.apache.kafka.connect.data.Struct,Object[]> window

mongo

private MongoDbConnection mongo

signallingCollectionId

private CollectionId signallingCollectionId

Constructor Details

MongoDbIncrementalSnapshotChangeEventSource

public MongoDbIncrementalSnapshotChangeEventSource(MongoDbConnectorConfig config,
 MongoDbTaskContext taskContext,
 MongoDbConnection.ChangeEventSourceConnectionFactory connections,
 ReplicaSets replicaSets,
 EventDispatcher<MongoDbPartition,CollectionId> dispatcher,
 MongoDbSchema collectionSchema,
 Clock clock,
 SnapshotProgressListener<MongoDbPartition> progressListener,
 DataChangeEventListener<MongoDbPartition> dataChangeEventListener)

Method Details

closeWindow

public void closeWindow(MongoDbPartition partition,
 String id,
 OffsetContext offsetContext)
                 throws InterruptedException

Specified by:

closeWindow in interface IncrementalSnapshotChangeEventSource<MongoDbPartition,CollectionId>

Throws:

InterruptedException

pauseSnapshot

public void pauseSnapshot(MongoDbPartition partition,
 OffsetContext offsetContext)
                   throws InterruptedException

Specified by:

pauseSnapshot in interface IncrementalSnapshotChangeEventSource<MongoDbPartition,CollectionId>

Throws:

InterruptedException

resumeSnapshot

public void resumeSnapshot(MongoDbPartition partition,
 OffsetContext offsetContext)
                    throws InterruptedException

Specified by:

resumeSnapshot in interface IncrementalSnapshotChangeEventSource<MongoDbPartition,CollectionId>

Throws:

InterruptedException

getSignalCollectionName

protected String getSignalCollectionName(String dataCollectionId)

sendWindowEvents

protected void sendWindowEvents(MongoDbPartition partition,
 OffsetContext offsetContext)
                         throws InterruptedException

Throws:

InterruptedException

sendEvent

protected void sendEvent(MongoDbPartition partition,
 EventDispatcher<MongoDbPartition,CollectionId> dispatcher,
 OffsetContext offsetContext,
 Object[] row)
                  throws InterruptedException

Throws:

InterruptedException

getChangeRecordEmitter

protected ChangeRecordEmitter<MongoDbPartition> getChangeRecordEmitter(MongoDbPartition partition,
 OffsetContext offsetContext,
 Object[] row)

Returns a ChangeRecordEmitter producing the change records for the given table row.

deduplicateWindow

protected void deduplicateWindow(io.debezium.spi.schema.DataCollectionId dataCollectionId,
 Object key)

emitWindowOpen

protected void emitWindowOpen()
                       throws InterruptedException

Update low watermark for the incremental snapshot chunk

Throws:

InterruptedException

emitWindowClose

protected void emitWindowClose()
                        throws InterruptedException

Update high watermark for the incremental snapshot chunk

Throws:

InterruptedException

init

public void init(MongoDbPartition partition,
 OffsetContext offsetContext)

Specified by:

init in interface IncrementalSnapshotChangeEventSource<MongoDbPartition,CollectionId>

readChunk

protected void readChunk(MongoDbPartition partition)
                  throws InterruptedException

Throws:

InterruptedException

nextDataCollection

private void nextDataCollection(MongoDbPartition partition)

readMaximumKey

private Object[] readMaximumKey()
                         throws InterruptedException

Throws:

InterruptedException

addDataCollectionNamesToSnapshot

public void addDataCollectionNamesToSnapshot(MongoDbPartition partition,
 List<String> dataCollectionIds,
 Optional<String> additionalCondition,
 Optional<String> surrogateKey,
 OffsetContext offsetContext)
                                      throws InterruptedException

Specified by:

addDataCollectionNamesToSnapshot in interface IncrementalSnapshotChangeEventSource<MongoDbPartition,CollectionId>

Throws:

InterruptedException

stopSnapshot

public void stopSnapshot(MongoDbPartition partition,
 List<String> dataCollectionIds,
 OffsetContext offsetContext)

Specified by:

stopSnapshot in interface IncrementalSnapshotChangeEventSource<MongoDbPartition,CollectionId>

createDataEventsForDataCollection

private void createDataEventsForDataCollection(MongoDbPartition partition)
                                        throws InterruptedException

Dispatches the data change events for the records of a single table.

Throws:

InterruptedException

incrementTableRowsScanned

private void incrementTableRowsScanned(long rows)

collectionScanCompleted

private void collectionScanCompleted(MongoDbPartition partition)

getTableScanLogTimer

private Threads.Timer getTableScanLogTimer()

keyFromRow

private Object[] keyFromRow(Object[] row)

setContext

protected void setContext(IncrementalSnapshotContext<CollectionId> context)

preReadChunk

protected void preReadChunk(IncrementalSnapshotContext<CollectionId> context)

postReadChunk

protected void postReadChunk(IncrementalSnapshotContext<CollectionId> context)

postIncrementalSnapshotCompleted

protected void postIncrementalSnapshotCompleted()

processMessage

public void processMessage(MongoDbPartition partition,
 io.debezium.spi.schema.DataCollectionId dataCollectionId,
 Object key,
 OffsetContext offsetContext)
                    throws InterruptedException

Specified by:

processMessage in interface IncrementalSnapshotChangeEventSource<MongoDbPartition,CollectionId>

Throws:

InterruptedException