Package io.debezium.connector.mongodb

Class MongoDbIncrementalSnapshotChangeEventSource

java.lang.Object

io.debezium.connector.mongodb.MongoDbIncrementalSnapshotChangeEventSource

All Implemented Interfaces:

IncrementalSnapshotChangeEventSource<MongoDbPartition,CollectionId>

@NotThreadSafe
public class MongoDbIncrementalSnapshotChangeEventSource
extends Object
implements IncrementalSnapshotChangeEventSource<MongoDbPartition,CollectionId>

An incremental snapshot change event source that emits events from a MongoDB change stream interleaved with snapshot events.

Field Summary

Fields

Modifier and Type	Field	Description
private static String	AUTHORIZATION_FAILURE_MESSAGE
private Clock	clock
private MongoDbSchema	collectionSchema
private ConnectionContext	connectionContext
private MongoDbConnectorConfig	connectorConfig
protected IncrementalSnapshotContext<CollectionId>	context
private MongoDbCollectionSchema	currentCollection
private DataChangeEventListener<MongoDbPartition>	dataListener
protected EventDispatcher<MongoDbPartition,CollectionId>	dispatcher
private static String	DOCUMENT_ID
private static org.slf4j.Logger	LOGGER
private ConnectionContext.MongoPrimary	primary
private SnapshotProgressListener<MongoDbPartition>	progressListener
private ReplicaSets	replicaSets
private CollectionId	signallingCollectionId
private MongoDbTaskContext	taskContext
private long	totalRowsScanned
protected Map<org.apache.kafka.connect.data.Struct,Object[]>	window

Constructor Summary

Constructors

Constructor	Description
MongoDbIncrementalSnapshotChangeEventSource(MongoDbConnectorConfig config, MongoDbTaskContext taskContext, ReplicaSets replicaSets, EventDispatcher<MongoDbPartition,CollectionId> dispatcher, MongoDbSchema collectionSchema, Clock clock, SnapshotProgressListener<MongoDbPartition> progressListener, DataChangeEventListener<MongoDbPartition> dataChangeEventListener)

Method Summary

Modifier and Type	Method	Description
void	addDataCollectionNamesToSnapshot(MongoDbPartition partition, List<String> dataCollectionIds, OffsetContext offsetContext)
void	closeWindow(MongoDbPartition partition, String id, OffsetContext offsetContext)
private void	collectionScanCompleted(MongoDbPartition partition)
private void	createDataEventsForDataCollection(MongoDbPartition partition)	Dispatches the data change events for the records of a single table.
protected void	deduplicateWindow(DataCollectionId dataCollectionId, Object key)
protected void	emitWindowClose()	Update high watermark for the incremental snapshot chunk
protected void	emitWindowOpen()	Update low watermark for the incremental snapshot chunk
private ConnectionContext.MongoPrimary	establishConnectionToPrimary(MongoDbPartition partition, ReplicaSet replicaSet)
protected ChangeRecordEmitter<MongoDbPartition>	getChangeRecordEmitter(MongoDbPartition partition, OffsetContext offsetContext, Object[] row)	Returns a ChangeRecordEmitter producing the change records for the given table row.
protected String	getSignalCollectionName(String dataCollectionId)
private Threads.Timer	getTableScanLogTimer()
private void	incrementTableRowsScanned(long rows)
void	init(MongoDbPartition partition, OffsetContext offsetContext)
private Object[]	keyFromRow(Object[] row)
private void	nextDataCollection(MongoDbPartition partition)
protected void	postIncrementalSnapshotCompleted()
protected void	postReadChunk(IncrementalSnapshotContext<CollectionId> context)
protected void	preReadChunk(IncrementalSnapshotContext<CollectionId> context)
void	processMessage(MongoDbPartition partition, DataCollectionId dataCollectionId, Object key, OffsetContext offsetContext)
protected void	readChunk(MongoDbPartition partition)
private Object[]	readMaximumKey()
protected void	sendEvent(MongoDbPartition partition, EventDispatcher<MongoDbPartition,CollectionId> dispatcher, OffsetContext offsetContext, Object[] row)
protected void	sendWindowEvents(MongoDbPartition partition, OffsetContext offsetContext)
protected void	setContext(IncrementalSnapshotContext<CollectionId> context)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface io.debezium.pipeline.source.snapshot.incremental.IncrementalSnapshotChangeEventSource

processFilteredEvent, processHeartbeat, processSchemaChange, processTransactionCommittedEvent, processTransactionStartedEvent

Field Detail

DOCUMENT_ID

private static final String DOCUMENT_ID

See Also:

Constant Field Values

LOGGER

private static final org.slf4j.Logger LOGGER

AUTHORIZATION_FAILURE_MESSAGE

private static final String AUTHORIZATION_FAILURE_MESSAGE

See Also:

Constant Field Values

connectorConfig

private final MongoDbConnectorConfig connectorConfig

clock

private final Clock clock

collectionSchema

private final MongoDbSchema collectionSchema

progressListener

private final SnapshotProgressListener<MongoDbPartition> progressListener

dataListener

private final DataChangeEventListener<MongoDbPartition> dataListener

totalRowsScanned

private long totalRowsScanned

replicaSets

private final ReplicaSets replicaSets

connectionContext

private final ConnectionContext connectionContext

taskContext

private final MongoDbTaskContext taskContext

currentCollection

private MongoDbCollectionSchema currentCollection

dispatcher

protected EventDispatcher<MongoDbPartition,CollectionId> dispatcher

context

protected IncrementalSnapshotContext<CollectionId> context

window

protected final Map<org.apache.kafka.connect.data.Struct,Object[]> window

primary

private ConnectionContext.MongoPrimary primary

signallingCollectionId

private CollectionId signallingCollectionId

Constructor Detail

MongoDbIncrementalSnapshotChangeEventSource

public MongoDbIncrementalSnapshotChangeEventSource(MongoDbConnectorConfig config,
                                                   MongoDbTaskContext taskContext,
                                                   ReplicaSets replicaSets,
                                                   EventDispatcher<MongoDbPartition,CollectionId> dispatcher,
                                                   MongoDbSchema collectionSchema,
                                                   Clock clock,
                                                   SnapshotProgressListener<MongoDbPartition> progressListener,
                                                   DataChangeEventListener<MongoDbPartition> dataChangeEventListener)

Method Detail

closeWindow

public void closeWindow(MongoDbPartition partition,
                        String id,
                        OffsetContext offsetContext)
                 throws InterruptedException

Specified by:

closeWindow in interface IncrementalSnapshotChangeEventSource<MongoDbPartition,CollectionId>

Throws:

InterruptedException

getSignalCollectionName

protected String getSignalCollectionName(String dataCollectionId)

sendWindowEvents

protected void sendWindowEvents(MongoDbPartition partition,
                                OffsetContext offsetContext)
                         throws InterruptedException

Throws:

InterruptedException

sendEvent

protected void sendEvent(MongoDbPartition partition,
                         EventDispatcher<MongoDbPartition,CollectionId> dispatcher,
                         OffsetContext offsetContext,
                         Object[] row)
                  throws InterruptedException

Throws:

InterruptedException

getChangeRecordEmitter

protected ChangeRecordEmitter<MongoDbPartition> getChangeRecordEmitter(MongoDbPartition partition,
                                                                       OffsetContext offsetContext,
                                                                       Object[] row)

Returns a ChangeRecordEmitter producing the change records for the given table row.

deduplicateWindow

protected void deduplicateWindow(DataCollectionId dataCollectionId,
                                 Object key)

emitWindowOpen

protected void emitWindowOpen()
                       throws InterruptedException

Update low watermark for the incremental snapshot chunk

Throws:

InterruptedException

emitWindowClose

protected void emitWindowClose()
                        throws InterruptedException

Update high watermark for the incremental snapshot chunk

Throws:

InterruptedException

init

public void init(MongoDbPartition partition,
                 OffsetContext offsetContext)

Specified by:

init in interface IncrementalSnapshotChangeEventSource<MongoDbPartition,CollectionId>

readChunk

protected void readChunk(MongoDbPartition partition)
                  throws InterruptedException

Throws:

InterruptedException

nextDataCollection

private void nextDataCollection(MongoDbPartition partition)

readMaximumKey

private Object[] readMaximumKey()
                         throws InterruptedException

Throws:

InterruptedException

addDataCollectionNamesToSnapshot

public void addDataCollectionNamesToSnapshot(MongoDbPartition partition,
                                             List<String> dataCollectionIds,
                                             OffsetContext offsetContext)
                                      throws InterruptedException

Specified by:

addDataCollectionNamesToSnapshot in interface IncrementalSnapshotChangeEventSource<MongoDbPartition,CollectionId>

Throws:

InterruptedException

createDataEventsForDataCollection

private void createDataEventsForDataCollection(MongoDbPartition partition)
                                        throws InterruptedException

Dispatches the data change events for the records of a single table.

Throws:

InterruptedException

incrementTableRowsScanned

private void incrementTableRowsScanned(long rows)

collectionScanCompleted

private void collectionScanCompleted(MongoDbPartition partition)

getTableScanLogTimer

private Threads.Timer getTableScanLogTimer()

keyFromRow

private Object[] keyFromRow(Object[] row)

setContext

protected void setContext(IncrementalSnapshotContext<CollectionId> context)

preReadChunk

protected void preReadChunk(IncrementalSnapshotContext<CollectionId> context)

postReadChunk

protected void postReadChunk(IncrementalSnapshotContext<CollectionId> context)

postIncrementalSnapshotCompleted

protected void postIncrementalSnapshotCompleted()

processMessage

public void processMessage(MongoDbPartition partition,
                           DataCollectionId dataCollectionId,
                           Object key,
                           OffsetContext offsetContext)
                    throws InterruptedException

Specified by:

processMessage in interface IncrementalSnapshotChangeEventSource<MongoDbPartition,CollectionId>

Throws:

InterruptedException

establishConnectionToPrimary

private ConnectionContext.MongoPrimary establishConnectionToPrimary(MongoDbPartition partition,
                                                                    ReplicaSet replicaSet)