Package io.debezium.connector.mysql

Class MySqlConnectorIT

java.lang.Object

io.debezium.embedded.AbstractConnectorTest

io.debezium.connector.mysql.MySqlConnectorIT

All Implemented Interfaces:

Testing

public class MySqlConnectorIT
extends AbstractConnectorTest

Author:

Randall Hauch

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
protected static class	MySqlConnectorIT.BinlogPosition
private static class	MySqlConnectorIT.NoTombStonesHandler

Nested classes/interfaces inherited from class io.debezium.embedded.AbstractConnectorTest

AbstractConnectorTest.SourceRecords

Nested classes/interfaces inherited from interface io.debezium.util.Testing

Testing.Debug, Testing.Files, Testing.InterruptableFunction, Testing.Network, Testing.Print, Testing.Timer

Field Summary

Fields

Modifier and Type	Field	Description
private Configuration	config
private final UniqueDatabase	DATABASE
private static final int	INITIAL_EVENT_COUNT
private static final int	ORDERS_TABLE_EVENT_COUNT
private static final int	PRODUCTS_TABLE_EVENT_COUNT
private final UniqueDatabase	RO_DATABASE
private static final Path	SCHEMA_HISTORY_PATH

Fields inherited from class io.debezium.embedded.AbstractConnectorTest

consumedLines, engine, logger, logTestName, OFFSET_STORE_PATH, pollTimeoutInMs, skipTestRule

Constructor Summary

Constructors

Constructor	Description
MySqlConnectorIT()

Method Summary

Modifier and Type	Method	Description
void	afterEach()
void	beforeEach()
private void	dropDatabases()
private org.apache.kafka.connect.data.Struct	getAfter(org.apache.kafka.connect.source.SourceRecord record)
private Optional<org.apache.kafka.connect.header.Header>	getHeaderField(org.apache.kafka.connect.source.SourceRecord record, String fieldName)
private Optional<org.apache.kafka.connect.header.Header>	getPKUpdateNewKeyHeader(org.apache.kafka.connect.source.SourceRecord record)
private Optional<org.apache.kafka.connect.header.Header>	getPKUpdateOldKeyHeader(org.apache.kafka.connect.source.SourceRecord record)
void	parseDeleteQuery()	This test case validates that if you enable MySQL option binlog_rows_query_log_events, then the original SQL statement for a DELETE over a single row is parsed into the resulting event.
void	parseMultipleInsertStatements()	This test case validates that if you enable MySQL option binlog_rows_query_log_events, then the issue multiple INSERTs, the appropriate SQL statements are parsed into the resulting events.
void	parseMultipleRowInsertStatement()	This test case validates that if you enable MySQL option binlog_rows_query_log_events, then the issue single multi-row INSERT, the appropriate SQL statements are parsed into the resulting events.
void	parseMultiRowDeleteQuery()	This test case validates that if you enable MySQL option binlog_rows_query_log_events, then issue a multi-row DELETE, the resulting events get the original SQL statement.
void	parseMultiRowUpdateQuery()	This test case validates that if you enable MySQL option binlog_rows_query_log_events, then the original SQL statement for an UPDATE over a single row is parsed into the resulting event.
void	parseUpdateQuery()	This test case validates that if you enable MySQL option binlog_rows_query_log_events, then the original SQL statement for an UPDATE over a single row is parsed into the resulting event.
private List<org.apache.kafka.connect.source.SourceRecord>	recordsForTopicForRoProductsTable(AbstractConnectorTest.SourceRecords records)
void	shouldConsumeAllEventsFromDatabaseUsingSnapshot()
private void	shouldConsumeAllEventsFromDatabaseUsingSnapshotByField(Field dbIncludeListField, int serverId)
void	shouldConsumeAllEventsFromDatabaseUsingSnapshotOld()
void	shouldConsumeEventsWithIncludedColumns()
void	shouldConsumeEventsWithIncludedColumnsForKeywordNamedTable()
void	shouldConsumeEventsWithMaskedAndBlacklistedColumns()
void	shouldConsumeEventsWithMaskedHashedColumns()
void	shouldConsumeEventsWithNoSnapshot()
void	shouldConsumeEventsWithTruncatedColumns()
void	shouldEmitHeadersOnPrimaryKeyUpdate()
void	shouldEmitNoEventsForSkippedCreateOperations()
void	shouldEmitNoEventsForSkippedUpdateAndDeleteOperations()
void	shouldEmitNoSavepoints()
void	shouldEmitNoTombstoneOnDelete()
void	shouldEmitTombstoneOnDeleteByDefault()
void	shouldEmitTruncateOperation()
void	shouldFailToValidateAdaptivePrecisionMode()	Specifying the adaptive time.precision.mode is no longer valid and a configuration validation problem should be reported when that configuration option is used.
void	shouldFailToValidateInvalidConfiguration()
void	shouldHandleIncludedTables()
void	shouldHandleIncludeListTables()
void	shouldIgnoreAlterTableForNonCapturedTablesNotStoredInHistory()
void	shouldIgnoreAlterTableForNonCapturedTablesStoredInHistory()
void	shouldIgnoreCreateIndexForNonCapturedTablesNotStoredInHistory()
void	shouldNotParseQueryIfConnectorNotConfiguredTo()	This test case validates that if you enable MySQL option binlog_rows_query_log_events, but configure the connector to NOT include the query, it will not be included in the event.
void	shouldNotParseQueryIfServerOptionDisabled()	This test case validates that if you disable MySQL option binlog_rows_query_log_events, then the original SQL statement for an INSERT statement is NOT parsed into the resulting event.
void	shouldNotSendTombstonesWhenNotSupportedByHandler()
void	shouldNotStartWithInvalidConfiguration()	Verifies that the connector doesn't run with an invalid configuration.
void	shouldOutputRecordsInCloudEventsFormat()
void	shouldParseQueryIfAvailableAndConnectorOptionEnabled()	This test case validates that if you enable MySQL option binlog_rows_query_log_events, then the original SQL statement for an INSERT statement is parsed into the resulting event.
void	shouldProcessCreateUniqueIndex()
void	shouldReceiveSchemaForNonWhitelistedTablesAndDatabases()
void	shouldRewriteIdentityKey()
void	shouldRewriteIdentityKeyWithMsgKeyColumnsFieldRegexValidation()
void	shouldRewriteIdentityKeyWithWhitespace()
void	shouldSaveSetCharacterSetWhenStoringOnlyCapturededTables()
void	shouldUseMultipleOverriddenSelectStatementsDuringSnapshotting()
void	shouldUseOverriddenSelectStatementDuringSnapshotting()
void	shouldValidateAcceptableConfiguration()
void	shouldValidateLockingModeNoneWithValidSnapshotModeConfiguration()	Validates that SNAPSHOT_LOCKING_MODE 'none' is valid with all snapshot modes
void	testDmlInChangeEvents()
void	testEmptySchemaLogWarningWithDatabaseWhitelist()
void	testEmptySchemaWarningWithTableWhitelist()
void	testNoEmptySchemaLogWarningWithDatabaseWhitelist()
void	testNoEmptySchemaLogWarningWithSnapshotNever()
void	testNoEmptySchemaWarningWithTableWhitelist()
private <T> void	validateConfigField(org.apache.kafka.common.config.Config config, Field field, T expectedValue)
private void	waitForStreamingRunning(String serverName)

Methods inherited from class io.debezium.embedded.AbstractConnectorTest

assertBeginTransaction, assertConfigurationErrors, assertConfigurationErrors, assertConfigurationErrors, assertConnectorIsRunning, assertConnectorNotRunning, assertDelete, assertEndTransaction, assertEngineIsRunning, assertHasNoSourceQuery, assertInsert, assertKey, assertNoConfigurationErrors, assertNoRecordsToConsume, assertOffset, assertOffset, assertOnlyTransactionRecordsToConsume, assertRecordTransactionMetadata, assertSchemaMatchesStruct, assertSchemaMatchesStruct, assertSourceQuery, assertTombstone, assertTombstone, assertUpdate, assertValueField, configValue, consumeAvailableRecords, consumeDmlRecordsByTopic, consumeDmlRecordsByTopic, consumeDmlRecordsByTopic, consumeRecord, consumeRecords, consumeRecords, consumeRecords, consumeRecordsButSkipUntil, consumeRecordsByTopic, consumeRecordsByTopic, consumeRecordsByTopic, consumeRecordsByTopicUntil, consumeRecordsUntil, debug, getConsumer, getMaximumEnqueuedRecordCount, getSnapshotMetricsObjectName, getSnapshotMetricsObjectName, getStreamingMetricsObjectName, getStreamingMetricsObjectName, getStreamingNamespace, initializeConnectorTestFramework, isStreamingRunning, isStreamingRunning, isTransactionRecord, loggingCompletion, print, readLastCommittedOffset, readLastCommittedOffsets, setConsumeTimeout, skipAvroValidation, start, start, start, start, start, start, start, startAndConsumeTillEnd, startAndConsumeTillEnd, stopConnector, stopConnector, validate, waitForAvailableRecords, waitForConnectorShutdown, waitForNotInitialState, waitForSnapshotToBeCompleted, waitForSnapshotToBeCompleted, waitForStreamingRunning, waitForStreamingRunning, waitTimeForRecords, waitTimeForRecordsAfterNulls

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface io.debezium.util.Testing

once, once, resetBeforeEachTest, time, time

Field Details

SCHEMA_HISTORY_PATH

private static final Path SCHEMA_HISTORY_PATH

DATABASE

private final UniqueDatabase DATABASE

RO_DATABASE

private final UniqueDatabase RO_DATABASE

PRODUCTS_TABLE_EVENT_COUNT

private static final int PRODUCTS_TABLE_EVENT_COUNT

See Also:

• Constant Field Values

ORDERS_TABLE_EVENT_COUNT

private static final int ORDERS_TABLE_EVENT_COUNT

See Also:

• Constant Field Values

INITIAL_EVENT_COUNT

private static final int INITIAL_EVENT_COUNT

See Also:

• Constant Field Values

config

private Configuration config

Constructor Details

MySqlConnectorIT

public MySqlConnectorIT()

Method Details

beforeEach

public void beforeEach()

afterEach

public void afterEach()

shouldNotStartWithInvalidConfiguration

public void shouldNotStartWithInvalidConfiguration()

Verifies that the connector doesn't run with an invalid configuration. This does not actually connect to the MySQL server.

shouldFailToValidateInvalidConfiguration

public void shouldFailToValidateInvalidConfiguration()

shouldValidateAcceptableConfiguration

public void shouldValidateAcceptableConfiguration()

validateConfigField

private <T> void validateConfigField(org.apache.kafka.common.config.Config config,
 Field field,
 T expectedValue)

shouldValidateLockingModeNoneWithValidSnapshotModeConfiguration

@FixFor("DBZ-639")
public void shouldValidateLockingModeNoneWithValidSnapshotModeConfiguration()

Validates that SNAPSHOT_LOCKING_MODE 'none' is valid with all snapshot modes

getPKUpdateNewKeyHeader

private Optional<org.apache.kafka.connect.header.Header> getPKUpdateNewKeyHeader(org.apache.kafka.connect.source.SourceRecord record)

getPKUpdateOldKeyHeader

private Optional<org.apache.kafka.connect.header.Header> getPKUpdateOldKeyHeader(org.apache.kafka.connect.source.SourceRecord record)

getHeaderField

private Optional<org.apache.kafka.connect.header.Header> getHeaderField(org.apache.kafka.connect.source.SourceRecord record,
 String fieldName)

shouldConsumeAllEventsFromDatabaseUsingSnapshot

public void shouldConsumeAllEventsFromDatabaseUsingSnapshot()
                                                     throws SQLException,
InterruptedException

Throws:

SQLException

InterruptedException

shouldConsumeAllEventsFromDatabaseUsingSnapshotOld

public void shouldConsumeAllEventsFromDatabaseUsingSnapshotOld()
                                                        throws SQLException,
InterruptedException

Throws:

SQLException

InterruptedException

shouldConsumeAllEventsFromDatabaseUsingSnapshotByField

private void shouldConsumeAllEventsFromDatabaseUsingSnapshotByField(Field dbIncludeListField,
 int serverId)
                                                             throws SQLException,
InterruptedException

Throws:

SQLException

InterruptedException

shouldUseOverriddenSelectStatementDuringSnapshotting

public void shouldUseOverriddenSelectStatementDuringSnapshotting()
                                                          throws SQLException,
InterruptedException

Throws:

SQLException

InterruptedException

shouldUseMultipleOverriddenSelectStatementsDuringSnapshotting

public void shouldUseMultipleOverriddenSelectStatementsDuringSnapshotting()
                                                                   throws SQLException,
InterruptedException

Throws:

SQLException

InterruptedException

shouldIgnoreAlterTableForNonCapturedTablesNotStoredInHistory

@FixFor("DBZ-977")
public void shouldIgnoreAlterTableForNonCapturedTablesNotStoredInHistory()
                                                                  throws SQLException,
InterruptedException

Throws:

SQLException

InterruptedException

shouldSaveSetCharacterSetWhenStoringOnlyCapturededTables

@FixFor("DBZ-1201")
public void shouldSaveSetCharacterSetWhenStoringOnlyCapturededTables()
                                                              throws SQLException,
InterruptedException

Throws:

SQLException

InterruptedException

shouldProcessCreateUniqueIndex

@FixFor("DBZ-1246")
public void shouldProcessCreateUniqueIndex()
                                    throws SQLException,
InterruptedException

Throws:

SQLException

InterruptedException

shouldIgnoreAlterTableForNonCapturedTablesStoredInHistory

@FixFor("DBZ-977")
public void shouldIgnoreAlterTableForNonCapturedTablesStoredInHistory()
                                                               throws SQLException,
InterruptedException

Throws:

SQLException

InterruptedException

shouldIgnoreCreateIndexForNonCapturedTablesNotStoredInHistory

@FixFor("DBZ-1264")
public void shouldIgnoreCreateIndexForNonCapturedTablesNotStoredInHistory()
                                                                   throws SQLException,
InterruptedException

Throws:

SQLException

InterruptedException

shouldReceiveSchemaForNonWhitelistedTablesAndDatabases

@FixFor("DBZ-683")
public void shouldReceiveSchemaForNonWhitelistedTablesAndDatabases()
                                                            throws SQLException,
InterruptedException

Throws:

SQLException

InterruptedException

shouldHandleIncludeListTables

@FixFor("DBZ-1546")
public void shouldHandleIncludeListTables()
                                   throws SQLException,
InterruptedException

Throws:

SQLException

InterruptedException

shouldHandleIncludedTables

public void shouldHandleIncludedTables()
                                throws SQLException,
InterruptedException

Throws:

SQLException

InterruptedException

dropDatabases

private void dropDatabases()
                    throws SQLException

Throws:

SQLException

getAfter

private org.apache.kafka.connect.data.Struct getAfter(org.apache.kafka.connect.source.SourceRecord record)

shouldConsumeEventsWithNoSnapshot

public void shouldConsumeEventsWithNoSnapshot()
                                       throws SQLException,
InterruptedException

Throws:

SQLException

InterruptedException

shouldConsumeEventsWithIncludedColumns

@FixFor("DBZ-1962")
public void shouldConsumeEventsWithIncludedColumns()
                                            throws SQLException,
InterruptedException

Throws:

SQLException

InterruptedException

shouldConsumeEventsWithIncludedColumnsForKeywordNamedTable

@FixFor("DBZ-2525")
public void shouldConsumeEventsWithIncludedColumnsForKeywordNamedTable()
                                                                throws SQLException,
InterruptedException

Throws:

SQLException

InterruptedException

shouldConsumeEventsWithMaskedAndBlacklistedColumns

public void shouldConsumeEventsWithMaskedAndBlacklistedColumns()
                                                        throws SQLException,
InterruptedException

Throws:

SQLException

InterruptedException

shouldConsumeEventsWithMaskedHashedColumns

@FixFor("DBZ-1692")
public void shouldConsumeEventsWithMaskedHashedColumns()
                                                throws InterruptedException

Throws:

InterruptedException

shouldConsumeEventsWithTruncatedColumns

@FixFor("DBZ-1972")
public void shouldConsumeEventsWithTruncatedColumns()
                                             throws InterruptedException

Throws:

InterruptedException

shouldEmitTombstoneOnDeleteByDefault

@FixFor("DBZ-582")
public void shouldEmitTombstoneOnDeleteByDefault()
                                          throws Exception

Throws:

Exception

shouldEmitNoTombstoneOnDelete

@FixFor("DBZ-582")
public void shouldEmitNoTombstoneOnDelete()
                                   throws Exception

Throws:

Exception

shouldEmitNoSavepoints

@FixFor("DBZ-794")
public void shouldEmitNoSavepoints()
                            throws Exception

Throws:

Exception

shouldNotParseQueryIfServerOptionDisabled

@FixFor("DBZ-706")
public void shouldNotParseQueryIfServerOptionDisabled()
                                               throws Exception

This test case validates that if you disable MySQL option binlog_rows_query_log_events, then the original SQL statement for an INSERT statement is NOT parsed into the resulting event.

Throws:

Exception

shouldNotParseQueryIfConnectorNotConfiguredTo

@FixFor("DBZ-706")
public void shouldNotParseQueryIfConnectorNotConfiguredTo()
                                                   throws Exception

This test case validates that if you enable MySQL option binlog_rows_query_log_events, but configure the connector to NOT include the query, it will not be included in the event.

Throws:

Exception

shouldParseQueryIfAvailableAndConnectorOptionEnabled

@FixFor("DBZ-706")
public void shouldParseQueryIfAvailableAndConnectorOptionEnabled()
                                                          throws Exception

This test case validates that if you enable MySQL option binlog_rows_query_log_events, then the original SQL statement for an INSERT statement is parsed into the resulting event.

Throws:

Exception

parseMultipleInsertStatements

@FixFor("DBZ-706")
public void parseMultipleInsertStatements()
                                   throws Exception

This test case validates that if you enable MySQL option binlog_rows_query_log_events, then the issue multiple INSERTs, the appropriate SQL statements are parsed into the resulting events.

Throws:

Exception

parseMultipleRowInsertStatement

@FixFor("DBZ-706")
public void parseMultipleRowInsertStatement()
                                     throws Exception

This test case validates that if you enable MySQL option binlog_rows_query_log_events, then the issue single multi-row INSERT, the appropriate SQL statements are parsed into the resulting events.

Throws:

Exception

parseDeleteQuery

@FixFor("DBZ-706")
public void parseDeleteQuery()
                      throws Exception

This test case validates that if you enable MySQL option binlog_rows_query_log_events, then the original SQL statement for a DELETE over a single row is parsed into the resulting event.

Throws:

Exception

parseMultiRowDeleteQuery

@FixFor("DBZ-706")
public void parseMultiRowDeleteQuery()
                              throws Exception

This test case validates that if you enable MySQL option binlog_rows_query_log_events, then issue a multi-row DELETE, the resulting events get the original SQL statement.

Throws:

Exception

parseUpdateQuery

@FixFor("DBZ-706")
public void parseUpdateQuery()
                      throws Exception

This test case validates that if you enable MySQL option binlog_rows_query_log_events, then the original SQL statement for an UPDATE over a single row is parsed into the resulting event.

Throws:

Exception

parseMultiRowUpdateQuery

@FixFor("DBZ-706")
public void parseMultiRowUpdateQuery()
                              throws Exception

This test case validates that if you enable MySQL option binlog_rows_query_log_events, then the original SQL statement for an UPDATE over a single row is parsed into the resulting event.

Throws:

Exception

shouldFailToValidateAdaptivePrecisionMode

@FixFor("DBZ-1234")
public void shouldFailToValidateAdaptivePrecisionMode()
                                               throws InterruptedException

Specifying the adaptive time.precision.mode is no longer valid and a configuration validation problem should be reported when that configuration option is used.

Throws:

InterruptedException

testEmptySchemaLogWarningWithDatabaseWhitelist

@FixFor("DBZ-1242")
public void testEmptySchemaLogWarningWithDatabaseWhitelist()
                                                    throws Exception

Throws:

Exception

testNoEmptySchemaLogWarningWithDatabaseWhitelist

@FixFor("DBZ-1242")
public void testNoEmptySchemaLogWarningWithDatabaseWhitelist()
                                                      throws Exception

Throws:

Exception

testEmptySchemaWarningWithTableWhitelist

@FixFor("DBZ-1242")
public void testEmptySchemaWarningWithTableWhitelist()
                                              throws Exception

Throws:

Exception

testNoEmptySchemaWarningWithTableWhitelist

@FixFor("DBZ-1242")
public void testNoEmptySchemaWarningWithTableWhitelist()
                                                throws Exception

Throws:

Exception

shouldRewriteIdentityKey

@FixFor("DBZ-1015")
public void shouldRewriteIdentityKey()
                              throws InterruptedException,
SQLException

Throws:

InterruptedException

SQLException

shouldRewriteIdentityKeyWithWhitespace

@FixFor("DBZ-2957")
public void shouldRewriteIdentityKeyWithWhitespace()
                                            throws InterruptedException,
SQLException

Throws:

InterruptedException

SQLException

shouldRewriteIdentityKeyWithMsgKeyColumnsFieldRegexValidation

@FixFor("DBZ-2957")
public void shouldRewriteIdentityKeyWithMsgKeyColumnsFieldRegexValidation()
                                                                   throws InterruptedException,
SQLException

Throws:

InterruptedException

SQLException

shouldOutputRecordsInCloudEventsFormat

@FixFor("DBZ-1292")
public void shouldOutputRecordsInCloudEventsFormat()
                                            throws Exception

Throws:

Exception

waitForStreamingRunning

private void waitForStreamingRunning(String serverName)
                              throws InterruptedException

Throws:

InterruptedException

recordsForTopicForRoProductsTable

private List<org.apache.kafka.connect.source.SourceRecord> recordsForTopicForRoProductsTable(AbstractConnectorTest.SourceRecords records)

shouldEmitHeadersOnPrimaryKeyUpdate

@FixFor("DBZ-1531")
public void shouldEmitHeadersOnPrimaryKeyUpdate()
                                         throws Exception

Throws:

Exception

shouldEmitNoEventsForSkippedCreateOperations

@FixFor("DBZ-1895")
public void shouldEmitNoEventsForSkippedCreateOperations()
                                                  throws Exception

Throws:

Exception

shouldEmitNoEventsForSkippedUpdateAndDeleteOperations

@FixFor("DBZ-1895")
public void shouldEmitNoEventsForSkippedUpdateAndDeleteOperations()
                                                           throws Exception

Throws:

Exception

testNoEmptySchemaLogWarningWithSnapshotNever

@FixFor("DBZ-1344")
public void testNoEmptySchemaLogWarningWithSnapshotNever()
                                                  throws Exception

Throws:

Exception

testDmlInChangeEvents

@FixFor("DBZ-3949")
public void testDmlInChangeEvents()
                           throws Exception

Throws:

Exception

shouldNotSendTombstonesWhenNotSupportedByHandler

@FixFor("DBZ-5052")
public void shouldNotSendTombstonesWhenNotSupportedByHandler()
                                                      throws Exception

Throws:

Exception

shouldEmitTruncateOperation

@FixFor("DBZ-5610")
public void shouldEmitTruncateOperation()
                                 throws Exception

Throws:

Exception