Package io.debezium.storage.kafka.history

Class KafkaSchemaHistory

java.lang.Object

io.debezium.relational.history.AbstractSchemaHistory

io.debezium.storage.kafka.history.KafkaSchemaHistory

All Implemented Interfaces:

SchemaHistory

@NotThreadSafe
public class KafkaSchemaHistory
extends AbstractSchemaHistory

A SchemaHistory implementation that records schema changes as normal SourceRecords on the specified topic, and that recovers the history by establishing a Kafka Consumer re-processing all messages on that topic.

Author:

Randall Hauch

Field Summary

Fields

Modifier and Type	Field	Description
static Field.Set	ALL_FIELDS
static final Field	BOOTSTRAP_SERVERS
private ExecutorService	checkTopicSettingsExecutor
private static final String	CLEANUP_POLICY_NAME
private static final String	CLEANUP_POLICY_VALUE
private static final String	CONSUMER_PREFIX
private Configuration	consumerConfig
private static final short	DEFAULT_TOPIC_REPLICATION_FACTOR	The default replication factor for the history topic which is used in case the value couldn't be retrieved from the broker.
private static final String	DEFAULT_TOPIC_REPLICATION_FACTOR_PROP_NAME	The name of broker property defining default replication factor for topics without the explicit setting.
static final Field	KAFKA_CREATE_TIMEOUT_MS
static final Field	KAFKA_QUERY_TIMEOUT_MS
private Duration	kafkaCreateTimeout
private Duration	kafkaQueryTimeout
private static final org.slf4j.Logger	LOGGER
private int	maxRecoveryAttempts
private static final Integer	PARTITION	The one and only partition of the history topic.
private static final int	PARTITION_COUNT
private Duration	pollInterval
private org.apache.kafka.clients.producer.KafkaProducer<String,String>	producer
private static final String	PRODUCER_PREFIX
private Configuration	producerConfig
private final DocumentReader	reader
static final Field	RECOVERY_POLL_ATTEMPTS
static final Field	RECOVERY_POLL_INTERVAL_MS
private static final String	RETENTION_BYTES_NAME
private static final long	RETENTION_MS_MAX
private static final long	RETENTION_MS_MIN
private static final String	RETENTION_MS_NAME
static final Field	TOPIC
private String	topicName
private static final int	UNLIMITED_VALUE
private static final boolean	USE_KAFKA_24_NEW_TOPIC_CONSTRUCTOR

Fields inherited from class io.debezium.relational.history.AbstractSchemaHistory

config, logger

Fields inherited from interface io.debezium.relational.history.SchemaHistory

CONFIGURATION_FIELD_PREFIX_STRING, DDL_FILTER, INTERNAL_CONNECTOR_CLASS, INTERNAL_CONNECTOR_ID, INTERNAL_PREFER_DDL, NAME, SKIP_UNPARSEABLE_DDL_STATEMENTS, STORE_ONLY_CAPTURED_DATABASES_DDL, STORE_ONLY_CAPTURED_TABLES_DDL

Constructor Summary

Constructors

Constructor	Description
KafkaSchemaHistory()

Method Summary

Modifier and Type	Method	Description
private void	checkTopicSettings(String topicName)
void	configure(Configuration config, HistoryRecordComparator comparator, SchemaHistoryListener listener, boolean useCatalogBeforeSchema)
static String	consumerConfigPropertyName(String kafkaConsumerPropertyName)
boolean	exists()
private static Field.Validator	forKafka(Field.Validator validator)
private short	getDefaultTopicReplicationFactor(org.apache.kafka.clients.admin.AdminClient admin)
private Long	getEndOffsetOfDbHistoryTopic(Long previousEndOffset, org.apache.kafka.clients.consumer.KafkaConsumer<String,String> historyConsumer)
private org.apache.kafka.clients.admin.Config	getKafkaBrokerConfig(org.apache.kafka.clients.admin.AdminClient admin)
private static boolean	hasNewTopicConstructorWithOptionals()
void	initializeStorage()
protected void	recoverRecords(Consumer<HistoryRecord> records)
void	start()
void	stop()
private void	stopCheckTopicSettingsExecutor()
boolean	storageExists()
protected void	storeRecord(HistoryRecord record)
String	toString()

Methods inherited from class io.debezium.relational.history.AbstractSchemaHistory

record, record, recover

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait

Methods inherited from interface io.debezium.relational.history.SchemaHistory

recover, recover

Field Details

LOGGER

private static final org.slf4j.Logger LOGGER

CLEANUP_POLICY_NAME

private static final String CLEANUP_POLICY_NAME

See Also:

• Constant Field Values

CLEANUP_POLICY_VALUE

private static final String CLEANUP_POLICY_VALUE

See Also:

• Constant Field Values

RETENTION_MS_NAME

private static final String RETENTION_MS_NAME

See Also:

• Constant Field Values

RETENTION_MS_MAX

private static final long RETENTION_MS_MAX

See Also:

• Constant Field Values

RETENTION_MS_MIN

private static final long RETENTION_MS_MIN

RETENTION_BYTES_NAME

private static final String RETENTION_BYTES_NAME

See Also:

• Constant Field Values

UNLIMITED_VALUE

private static final int UNLIMITED_VALUE

See Also:

• Constant Field Values

PARTITION_COUNT

private static final int PARTITION_COUNT

See Also:

• Constant Field Values

DEFAULT_TOPIC_REPLICATION_FACTOR_PROP_NAME

private static final String DEFAULT_TOPIC_REPLICATION_FACTOR_PROP_NAME

The name of broker property defining default replication factor for topics without the explicit setting.

See Also:

• kafka.server.KafkaConfig.DefaultReplicationFactorProp
• Constant Field Values

DEFAULT_TOPIC_REPLICATION_FACTOR

private static final short DEFAULT_TOPIC_REPLICATION_FACTOR

The default replication factor for the history topic which is used in case the value couldn't be retrieved from the broker.

See Also:

• Constant Field Values

TOPIC

public static final Field TOPIC

BOOTSTRAP_SERVERS

public static final Field BOOTSTRAP_SERVERS

RECOVERY_POLL_INTERVAL_MS

public static final Field RECOVERY_POLL_INTERVAL_MS

RECOVERY_POLL_ATTEMPTS

public static final Field RECOVERY_POLL_ATTEMPTS

KAFKA_QUERY_TIMEOUT_MS

public static final Field KAFKA_QUERY_TIMEOUT_MS

KAFKA_CREATE_TIMEOUT_MS

public static final Field KAFKA_CREATE_TIMEOUT_MS

ALL_FIELDS

public static Field.Set ALL_FIELDS

CONSUMER_PREFIX

private static final String CONSUMER_PREFIX

See Also:

• Constant Field Values

PRODUCER_PREFIX

private static final String PRODUCER_PREFIX

See Also:

• Constant Field Values

PARTITION

private static final Integer PARTITION

The one and only partition of the history topic.

reader

private final DocumentReader reader

topicName

private String topicName

consumerConfig

private Configuration consumerConfig

producerConfig

private Configuration producerConfig

producer

private volatile org.apache.kafka.clients.producer.KafkaProducer<String,String> producer

maxRecoveryAttempts

private int maxRecoveryAttempts

pollInterval

private Duration pollInterval

checkTopicSettingsExecutor

private ExecutorService checkTopicSettingsExecutor

kafkaQueryTimeout

private Duration kafkaQueryTimeout

kafkaCreateTimeout

private Duration kafkaCreateTimeout

USE_KAFKA_24_NEW_TOPIC_CONSTRUCTOR

private static final boolean USE_KAFKA_24_NEW_TOPIC_CONSTRUCTOR

Constructor Details

KafkaSchemaHistory

public KafkaSchemaHistory()

Method Details

hasNewTopicConstructorWithOptionals

private static boolean hasNewTopicConstructorWithOptionals()

configure

public void configure(Configuration config,
 HistoryRecordComparator comparator,
 SchemaHistoryListener listener,
 boolean useCatalogBeforeSchema)

Specified by:

configure in interface SchemaHistory

Overrides:

configure in class AbstractSchemaHistory

start

public void start()

Specified by:

start in interface SchemaHistory

Overrides:

start in class AbstractSchemaHistory

storeRecord

protected void storeRecord(HistoryRecord record)
                    throws SchemaHistoryException

Specified by:

storeRecord in class AbstractSchemaHistory

Throws:

SchemaHistoryException

recoverRecords

protected void recoverRecords(Consumer<HistoryRecord> records)

Specified by:

recoverRecords in class AbstractSchemaHistory

getEndOffsetOfDbHistoryTopic

private Long getEndOffsetOfDbHistoryTopic(Long previousEndOffset,
 org.apache.kafka.clients.consumer.KafkaConsumer<String,String> historyConsumer)

storageExists

public boolean storageExists()

exists

public boolean exists()

checkTopicSettings

private void checkTopicSettings(String topicName)

stop

public void stop()

Specified by:

stop in interface SchemaHistory

Overrides:

stop in class AbstractSchemaHistory

stopCheckTopicSettingsExecutor

private void stopCheckTopicSettingsExecutor()

toString

public String toString()

Overrides:

toString in class Object

consumerConfigPropertyName

public static String consumerConfigPropertyName(String kafkaConsumerPropertyName)

initializeStorage

public void initializeStorage()

Specified by:

initializeStorage in interface SchemaHistory

Overrides:

initializeStorage in class AbstractSchemaHistory

getDefaultTopicReplicationFactor

private short getDefaultTopicReplicationFactor(org.apache.kafka.clients.admin.AdminClient admin)
                                        throws Exception

Throws:

Exception

getKafkaBrokerConfig

private org.apache.kafka.clients.admin.Config getKafkaBrokerConfig(org.apache.kafka.clients.admin.AdminClient admin)
                                                            throws Exception

Throws:

Exception

forKafka

private static Field.Validator forKafka(Field.Validator validator)