Package io.debezium.connector.mongodb

Class MongoDbConnectorIT

java.lang.Object

io.debezium.embedded.AbstractConnectorTest

io.debezium.connector.mongodb.MongoDbConnectorIT

All Implemented Interfaces:

Testing

public class MongoDbConnectorIT
extends AbstractConnectorTest

Author:

Randall Hauch

Nested Class Summary

Nested classes/interfaces inherited from class io.debezium.embedded.AbstractConnectorTest

AbstractConnectorTest.SourceRecords

Nested classes/interfaces inherited from interface io.debezium.util.Testing

Testing.Debug, Testing.Files, Testing.InterruptableFunction, Testing.Network, Testing.Print, Testing.Timer

Field Summary

Fields

Modifier and Type	Field	Description
private Configuration	config
private io.debezium.connector.mongodb.MongoDbTaskContext	context

Fields inherited from class io.debezium.embedded.AbstractConnectorTest

engine, logger, logTestName, OFFSET_STORE_PATH, pollTimeoutInMs, skipTestRule

Constructor Summary

Constructors

Constructor	Description
MongoDbConnectorIT()

Method Summary

Modifier and Type	Method	Description
void	afterEach()
private static void	assertSourceRecordKeyFieldIsEqualTo(org.apache.kafka.connect.source.SourceRecord record, String fieldName, String expected)
void	beforeEach()
protected BiConsumer<String,Throwable>	connectionErrorHandler(int numErrorsBeforeFailing)
private void	deleteDocument(String dbName, String collectionName, org.bson.types.ObjectId objectId)
private String	formatObjectId(org.bson.types.ObjectId objId)
private void	insertDocuments(String dbName, String collectionName, org.bson.Document... documents)
protected List<org.bson.Document>	loadTestDocuments(String pathOnClasspath)
protected io.debezium.connector.mongodb.ConnectionContext.MongoPrimary	primary()
void	shouldConsumeAllEventsFromDatabase()
void	shouldConsumeAllEventsFromDatabaseWithCustomAuthSource()
void	shouldConsumeAllEventsFromDatabaseWithSkippedOperations()
void	shouldConsumeEventsFromCollectionWithReplacedTopicName()
void	shouldConsumeTransaction()
void	shouldEmitHeartbeatMessages()
void	shouldFailToValidateInvalidConfiguration()
void	shouldFilterItemsInCollectionWhileTakingSnapshot()
void	shouldGenerateRecordForDeleteEventWithoutTombstone()
void	shouldGenerateRecordForInsertEvent()
void	shouldGenerateRecordForUpdateEvent()
void	shouldGenerateRecordForUpdateEventUsingLegacyV1SourceInfo()
void	shouldGenerateRecordsWithCorrectlySerializedId()
void	shouldGeneratorRecordForDeleteEvent()
void	shouldNotReplicateSnapshot()
void	shouldNotStartWithInvalidConfiguration()	Verifies that the connector doesn't run with an invalid configuration.
void	shouldNotThrowExceptionWhenFieldExcludeListHasLeadingWhiteSpaces()
void	shouldNotThrowExceptionWhenFieldExcludeListHasWhiteSpaces()
void	shouldNotThrowExceptionWhenFieldExcludeListIsValid()
void	shouldNotThrowExceptionWhenFieldRenamesHasLeadingWhiteSpaces()
void	shouldNotThrowExceptionWhenFieldRenamesHasWhiteSpaces()
void	shouldNotThrowExceptionWhenFieldRenamesIsValid()
void	shouldOutputRecordsInCloudEventsFormat()
void	shouldReplicateContent()
void	shouldResumeTransactionInMiddle()
void	shouldSelectivelySnapshot()
void	shouldSnapshotDocumentContainingFieldNamedOp()
void	shouldSupportDbRef()
void	shouldSupportDbRef2()
void	shouldThrowExceptionWhenFieldExcludeListDatabaseAndCollectionPartIsOnlyProvided()
void	shouldThrowExceptionWhenFieldExcludeListDatabaseAndCollectionPartsAreMissing()
void	shouldThrowExceptionWhenFieldExcludeListDatabasePartIsOnlyProvided()
void	shouldThrowExceptionWhenFieldExcludeListFieldPartIsMissing()
void	shouldThrowExceptionWhenFieldRenamesDatabaseAndCollectionPartsAreMissing()
void	shouldThrowExceptionWhenFieldRenamesRenameMappingKeyIsMissing()
void	shouldThrowExceptionWhenFieldRenamesRenameMappingValueIsMissing()
void	shouldThrowExceptionWhenFieldRenamesReplacementPartIsMissing()
void	shouldThrowExceptionWhenFieldRenamesReplacementPartSeparatorIsMissing()
void	shouldUseSSL()
void	shouldValidateAcceptableConfiguration()
void	shouldValidateFilterFieldConfiguration(Field field, String value, int errorCount)
protected void	storeDocuments(com.mongodb.client.ClientSession session, com.mongodb.client.MongoCollection<org.bson.Document> collection, String pathOnClasspath)
protected void	storeDocuments(com.mongodb.client.MongoCollection<org.bson.Document> collection, String pathOnClasspath)
protected void	storeDocuments(String dbName, String collectionName, String pathOnClasspath)
protected void	storeDocumentsInTx(String dbName, String collectionName, String pathOnClasspath)
void	testEmptySchemaWarningAfterApplyingCollectionFilters()
private org.bson.types.ObjectId	toObjectId(String oid)
private void	updateDocuments(String dbName, String collectionName, org.bson.Document filter, org.bson.Document document)
protected void	verifyCreateOperation(org.apache.kafka.connect.source.SourceRecord record)
protected void	verifyDeleteOperation(org.apache.kafka.connect.source.SourceRecord record)
protected void	verifyFromInitialSync(org.apache.kafka.connect.source.SourceRecord record, AtomicBoolean foundLast)
protected void	verifyFromTransaction(org.apache.kafka.connect.source.SourceRecord record, long order)
protected void	verifyNotFromInitialSync(org.apache.kafka.connect.source.SourceRecord record)
protected void	verifyNotFromTransaction(org.apache.kafka.connect.source.SourceRecord record)
protected void	verifyOperation(org.apache.kafka.connect.source.SourceRecord record, Envelope.Operation expected)
protected void	verifyReadOperation(org.apache.kafka.connect.source.SourceRecord record)
protected void	verifyUpdateOperation(org.apache.kafka.connect.source.SourceRecord record)

Methods inherited from class io.debezium.embedded.AbstractConnectorTest

assertBeginTransaction, assertConfigurationErrors, assertConfigurationErrors, assertConfigurationErrors, assertConnectorIsRunning, assertConnectorNotRunning, assertDelete, assertEndTransaction, assertEngineIsRunning, assertHasNoSourceQuery, assertInsert, assertKey, assertNoConfigurationErrors, assertNoRecordsToConsume, assertOffset, assertOffset, assertOnlyTransactionRecordsToConsume, assertRecordTransactionMetadata, assertSchemaMatchesStruct, assertSchemaMatchesStruct, assertSourceQuery, assertTombstone, assertTombstone, assertUpdate, assertValueField, configValue, consumeAvailableRecords, consumeDmlRecordsByTopic, consumeDmlRecordsByTopic, consumeDmlRecordsByTopic, consumeRecord, consumeRecords, consumeRecords, consumeRecords, consumeRecordsByTopic, consumeRecordsByTopic, consumeRecordsByTopic, debug, getMaximumEnqueuedRecordCount, getSnapshotMetricsObjectName, getStreamingMetricsObjectName, getStreamingMetricsObjectName, getStreamingNamespace, initializeConnectorTestFramework, isStreamingRunning, isStreamingRunning, isTransactionRecord, loggingCompletion, print, readLastCommittedOffset, readLastCommittedOffsets, setConsumeTimeout, skipAvroValidation, start, start, start, start, start, startAndConsumeTillEnd, stopConnector, stopConnector, validate, waitForAvailableRecords, waitForConnectorShutdown, waitForSnapshotToBeCompleted, waitForStreamingRunning, waitForStreamingRunning, waitTimeForRecords, waitTimeForRecordsAfterNulls

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface io.debezium.util.Testing

once, once, resetBeforeEachTest, time, time

Field Detail

config

private Configuration config

context

private io.debezium.connector.mongodb.MongoDbTaskContext context

Constructor Detail

MongoDbConnectorIT

public MongoDbConnectorIT()

Method Detail

beforeEach

public void beforeEach()

afterEach

public void afterEach()

shouldNotStartWithInvalidConfiguration

public void shouldNotStartWithInvalidConfiguration()

Verifies that the connector doesn't run with an invalid configuration. This does not actually connect to the MySQL server.

shouldFailToValidateInvalidConfiguration

public void shouldFailToValidateInvalidConfiguration()

shouldThrowExceptionWhenFieldExcludeListDatabasePartIsOnlyProvided

public void shouldThrowExceptionWhenFieldExcludeListDatabasePartIsOnlyProvided()

shouldThrowExceptionWhenFieldExcludeListDatabaseAndCollectionPartIsOnlyProvided

public void shouldThrowExceptionWhenFieldExcludeListDatabaseAndCollectionPartIsOnlyProvided()

shouldThrowExceptionWhenFieldExcludeListDatabaseAndCollectionPartsAreMissing

public void shouldThrowExceptionWhenFieldExcludeListDatabaseAndCollectionPartsAreMissing()

shouldThrowExceptionWhenFieldExcludeListFieldPartIsMissing

public void shouldThrowExceptionWhenFieldExcludeListFieldPartIsMissing()

shouldNotThrowExceptionWhenFieldExcludeListHasLeadingWhiteSpaces

public void shouldNotThrowExceptionWhenFieldExcludeListHasLeadingWhiteSpaces()

shouldNotThrowExceptionWhenFieldExcludeListHasWhiteSpaces

public void shouldNotThrowExceptionWhenFieldExcludeListHasWhiteSpaces()

shouldNotThrowExceptionWhenFieldExcludeListIsValid

public void shouldNotThrowExceptionWhenFieldExcludeListIsValid()

shouldThrowExceptionWhenFieldRenamesDatabaseAndCollectionPartsAreMissing

public void shouldThrowExceptionWhenFieldRenamesDatabaseAndCollectionPartsAreMissing()

shouldThrowExceptionWhenFieldRenamesReplacementPartIsMissing

public void shouldThrowExceptionWhenFieldRenamesReplacementPartIsMissing()

shouldThrowExceptionWhenFieldRenamesReplacementPartSeparatorIsMissing

public void shouldThrowExceptionWhenFieldRenamesReplacementPartSeparatorIsMissing()

shouldThrowExceptionWhenFieldRenamesRenameMappingKeyIsMissing

public void shouldThrowExceptionWhenFieldRenamesRenameMappingKeyIsMissing()

shouldThrowExceptionWhenFieldRenamesRenameMappingValueIsMissing

public void shouldThrowExceptionWhenFieldRenamesRenameMappingValueIsMissing()

shouldNotThrowExceptionWhenFieldRenamesHasLeadingWhiteSpaces

public void shouldNotThrowExceptionWhenFieldRenamesHasLeadingWhiteSpaces()

shouldNotThrowExceptionWhenFieldRenamesHasWhiteSpaces

public void shouldNotThrowExceptionWhenFieldRenamesHasWhiteSpaces()

shouldNotThrowExceptionWhenFieldRenamesIsValid

public void shouldNotThrowExceptionWhenFieldRenamesIsValid()

shouldValidateFilterFieldConfiguration

public void shouldValidateFilterFieldConfiguration(Field field,
                                                   String value,
                                                   int errorCount)

shouldValidateAcceptableConfiguration

public void shouldValidateAcceptableConfiguration()

shouldConsumeAllEventsFromDatabase

public void shouldConsumeAllEventsFromDatabase()
                                        throws InterruptedException,
                                               IOException

Throws:

InterruptedException

IOException

shouldConsumeAllEventsFromDatabaseWithSkippedOperations

@FixFor("DBZ-1831")
public void shouldConsumeAllEventsFromDatabaseWithSkippedOperations()
                                                             throws InterruptedException,
                                                                    IOException

Throws:

InterruptedException

IOException

shouldConsumeAllEventsFromDatabaseWithCustomAuthSource

@FixFor("DBZ-1168")
public void shouldConsumeAllEventsFromDatabaseWithCustomAuthSource()
                                                            throws InterruptedException,
                                                                   IOException

Throws:

InterruptedException

IOException

shouldSupportDbRef

@FixFor("DBZ-1767")
public void shouldSupportDbRef()
                        throws InterruptedException,
                               IOException

Throws:

InterruptedException

IOException

shouldConsumeEventsFromCollectionWithReplacedTopicName

@FixFor("DBZ-865 and DBZ-1242")
public void shouldConsumeEventsFromCollectionWithReplacedTopicName()
                                                            throws InterruptedException,
                                                                   IOException

Throws:

InterruptedException

IOException

testEmptySchemaWarningAfterApplyingCollectionFilters

@FixFor("DBZ-1242")
public void testEmptySchemaWarningAfterApplyingCollectionFilters()
                                                          throws Exception

Throws:

Exception

verifyFromInitialSync

protected void verifyFromInitialSync(org.apache.kafka.connect.source.SourceRecord record,
                                     AtomicBoolean foundLast)

shouldConsumeTransaction

@FixFor("DBZ-1215")
public void shouldConsumeTransaction()
                              throws InterruptedException,
                                     IOException

Throws:

InterruptedException

IOException

shouldResumeTransactionInMiddle

@FixFor("DBZ-1215")
public void shouldResumeTransactionInMiddle()
                                     throws InterruptedException,
                                            IOException

Throws:

InterruptedException

IOException

shouldSnapshotDocumentContainingFieldNamedOp

@FixFor("DBZ-2116")
public void shouldSnapshotDocumentContainingFieldNamedOp()
                                                  throws Exception

Throws:

Exception

shouldFilterItemsInCollectionWhileTakingSnapshot

@FixFor("DBZ-2496")
public void shouldFilterItemsInCollectionWhileTakingSnapshot()
                                                      throws Exception

Throws:

Exception

shouldSelectivelySnapshot

@FixFor("DBZ-2456")
public void shouldSelectivelySnapshot()
                               throws InterruptedException

Throws:

InterruptedException

verifyNotFromInitialSync

protected void verifyNotFromInitialSync(org.apache.kafka.connect.source.SourceRecord record)

verifyFromTransaction

protected void verifyFromTransaction(org.apache.kafka.connect.source.SourceRecord record,
                                     long order)

verifyNotFromTransaction

protected void verifyNotFromTransaction(org.apache.kafka.connect.source.SourceRecord record)

verifyCreateOperation

protected void verifyCreateOperation(org.apache.kafka.connect.source.SourceRecord record)

verifyReadOperation

protected void verifyReadOperation(org.apache.kafka.connect.source.SourceRecord record)

verifyUpdateOperation

protected void verifyUpdateOperation(org.apache.kafka.connect.source.SourceRecord record)

verifyDeleteOperation

protected void verifyDeleteOperation(org.apache.kafka.connect.source.SourceRecord record)

verifyOperation

protected void verifyOperation(org.apache.kafka.connect.source.SourceRecord record,
                               Envelope.Operation expected)

primary

protected io.debezium.connector.mongodb.ConnectionContext.MongoPrimary primary()

storeDocuments

protected void storeDocuments(String dbName,
                              String collectionName,
                              String pathOnClasspath)

storeDocuments

protected void storeDocuments(com.mongodb.client.MongoCollection<org.bson.Document> collection,
                              String pathOnClasspath)

storeDocumentsInTx

protected void storeDocumentsInTx(String dbName,
                                  String collectionName,
                                  String pathOnClasspath)

storeDocuments

protected void storeDocuments(com.mongodb.client.ClientSession session,
                              com.mongodb.client.MongoCollection<org.bson.Document> collection,
                              String pathOnClasspath)

loadTestDocuments

protected List<org.bson.Document> loadTestDocuments(String pathOnClasspath)

connectionErrorHandler

protected BiConsumer<String,Throwable> connectionErrorHandler(int numErrorsBeforeFailing)

shouldUseSSL

public void shouldUseSSL()
                  throws InterruptedException,
                         IOException

Throws:

InterruptedException

IOException

shouldEmitHeartbeatMessages

@FixFor("DBZ-1198")
public void shouldEmitHeartbeatMessages()
                                 throws InterruptedException,
                                        IOException

Throws:

InterruptedException

IOException

shouldOutputRecordsInCloudEventsFormat

@FixFor("DBZ-1292")
public void shouldOutputRecordsInCloudEventsFormat()
                                            throws Exception

Throws:

Exception

shouldGenerateRecordForInsertEvent

public void shouldGenerateRecordForInsertEvent()
                                        throws Exception

Throws:

Exception

shouldGenerateRecordForUpdateEvent

public void shouldGenerateRecordForUpdateEvent()
                                        throws Exception

Throws:

Exception

shouldGeneratorRecordForDeleteEvent

public void shouldGeneratorRecordForDeleteEvent()
                                         throws Exception

Throws:

Exception

shouldGenerateRecordForDeleteEventWithoutTombstone

@FixFor("DBZ-582")
public void shouldGenerateRecordForDeleteEventWithoutTombstone()
                                                        throws Exception

Throws:

Exception

shouldGenerateRecordsWithCorrectlySerializedId

public void shouldGenerateRecordsWithCorrectlySerializedId()
                                                    throws Exception

Throws:

Exception

assertSourceRecordKeyFieldIsEqualTo

private static void assertSourceRecordKeyFieldIsEqualTo(org.apache.kafka.connect.source.SourceRecord record,
                                                        String fieldName,
                                                        String expected)

shouldSupportDbRef2

public void shouldSupportDbRef2()
                         throws Exception

Throws:

Exception

shouldReplicateContent

public void shouldReplicateContent()
                            throws Exception

Throws:

Exception

shouldNotReplicateSnapshot

public void shouldNotReplicateSnapshot()
                                throws Exception

Throws:

Exception

shouldGenerateRecordForUpdateEventUsingLegacyV1SourceInfo

@FixFor("DBZ-1880")
public void shouldGenerateRecordForUpdateEventUsingLegacyV1SourceInfo()
                                                               throws Exception

Throws:

Exception

formatObjectId

private String formatObjectId(org.bson.types.ObjectId objId)

insertDocuments

private void insertDocuments(String dbName,
                             String collectionName,
                             org.bson.Document... documents)

updateDocuments

private void updateDocuments(String dbName,
                             String collectionName,
                             org.bson.Document filter,
                             org.bson.Document document)

deleteDocument

private void deleteDocument(String dbName,
                            String collectionName,
                            org.bson.types.ObjectId objectId)

toObjectId

private org.bson.types.ObjectId toObjectId(String oid)