Package io.debezium.config

Class CommonConnectorConfig

java.lang.Object

io.debezium.config.CommonConnectorConfig

Direct Known Subclasses:

RelationalDatabaseConnectorConfig

public abstract class CommonConnectorConfig
extends Object

Configuration options common to all Debezium connectors.

Author:

Gunnar Morling

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
static class	CommonConnectorConfig.BinaryHandlingMode	The set of predefined BinaryHandlingMode options or aliases
static class	CommonConnectorConfig.EventProcessingFailureHandlingMode	The set of predefined modes for dealing with failures during event processing.
static class	CommonConnectorConfig.Version	The set of predefined versions e.g.

Field Summary

Fields

Modifier and Type	Field	Description
private static String	APICURIO_AVRO_CONVERTER
static Field	BINARY_HANDLING_MODE
private CommonConnectorConfig.BinaryHandlingMode	binaryHandlingMode
private Configuration	config
protected static ConfigDefinition	CONFIG_DEFINITION
private static String	CONFLUENT_AVRO_CONVERTER
private static String	CONVERTER_TYPE_SUFFIX
static Field	CUSTOM_CONVERTERS
static Field	CUSTOM_RETRIABLE_EXCEPTION
private CustomConverterRegistry	customConverterRegistry
static String	DATABASE_CONFIG_PREFIX
static int	DEFAULT_MAX_BATCH_SIZE
static int	DEFAULT_MAX_QUEUE_SIZE
static long	DEFAULT_MAX_QUEUE_SIZE_IN_BYTES
static long	DEFAULT_POLL_INTERVAL_MILLIS
static int	DEFAULT_QUERY_FETCH_SIZE
static long	DEFAULT_RETRIABLE_RESTART_WAIT
private boolean	emitTombstoneOnDelete
static Field	EVENT_PROCESSING_FAILURE_HANDLING_MODE
private CommonConnectorConfig.EventProcessingFailureHandlingMode	eventProcessingFailureHandlingMode
private Duration	heartbeatInterval
private String	heartbeatTopicsPrefix
static Field	INCREMENTAL_SNAPSHOT_ALLOW_SCHEMA_CHANGES
static Field	INCREMENTAL_SNAPSHOT_CHUNK_SIZE
private boolean	incrementalSnapshotAllowSchemaChanges
private int	incrementalSnapshotChunkSize
private static org.slf4j.Logger	LOGGER
private String	logicalName
static Field	MAX_BATCH_SIZE
static Field	MAX_QUEUE_SIZE
static Field	MAX_QUEUE_SIZE_IN_BYTES
private int	maxBatchSize
private int	maxQueueSize
private long	maxQueueSizeInBytes
static Field	POLL_INTERVAL_MS
private Duration	pollInterval
static Field	PROVIDE_TRANSACTION_METADATA
static Field	QUERY_FETCH_SIZE
private Integer	queryFetchSize
static Field	RETRIABLE_RESTART_WAIT
private Duration	retriableRestartWait
static Field	SANITIZE_FIELD_NAMES
private boolean	sanitizeFieldNames
private boolean	shouldProvideTransactionMetadata
static Field	SIGNAL_DATA_COLLECTION
private String	signalingDataCollection
static Field	SKIPPED_OPERATIONS
private EnumSet<Envelope.Operation>	skippedOperations
static Field	SNAPSHOT_DELAY_MS
static Field	SNAPSHOT_FETCH_SIZE
static Field	SNAPSHOT_MAX_THREADS
static Field	SNAPSHOT_MODE_TABLES
private Duration	snapshotDelay
private int	snapshotFetchSize
private int	snapshotMaxThreads
static Field	SOURCE_STRUCT_MAKER_VERSION
private SourceInfoStructMaker<? extends AbstractSourceInfo>	sourceInfoStructMaker
static String	TASK_ID
private String	taskId
static Field	TOMBSTONES_ON_DELETE
static Field	TRANSACTION_TOPIC
private String	transactionTopic

Constructor Summary

Constructors

Modifier	Constructor	Description
protected	CommonConnectorConfig(Configuration config, String logicalName, int defaultSnapshotFetchSize)

Method Summary

Modifier and Type	Method	Description
CommonConnectorConfig.BinaryHandlingMode	binaryHandlingMode()
CustomConverterRegistry	customConverterRegistry()
Optional<String>	customRetriableException()
private static EnumSet<Envelope.Operation>	determineSkippedOperations(Configuration config)
Configuration	getConfig()	Deprecated.
abstract String	getConnectorName()
abstract String	getContextName()
private List<CustomConverter<org.apache.kafka.connect.data.SchemaBuilder,ConvertedField>>	getCustomConverters()
Set<Pattern>	getDataCollectionsToBeSnapshotted()
CommonConnectorConfig.EventProcessingFailureHandlingMode	getEventProcessingFailureHandlingMode()
Duration	getHeartbeatInterval()
String	getHeartbeatTopicsPrefix()
int	getIncrementalSnashotChunkSize()
String	getLogicalName()
int	getMaxBatchSize()
int	getMaxQueueSize()
long	getMaxQueueSizeInBytes()
Duration	getPollInterval()
int	getQueryFetchSize()
Duration	getRetriableRestartWait()
boolean	getSanitizeFieldNames()
String	getSignalingDataCollectionId()
EnumSet<Envelope.Operation>	getSkippedOperations()
Duration	getSnapshotDelay()
int	getSnapshotFetchSize()
int	getSnapshotMaxThreads()
<T extends AbstractSourceInfo> SourceInfoStructMaker<T>	getSourceInfoStructMaker()
protected abstract SourceInfoStructMaker<?>	getSourceInfoStructMaker(CommonConnectorConfig.Version version)	Returns the connector-specific SourceInfoStructMaker based on the given configuration.
String	getTaskId()
String	getTransactionTopic()	Returns the name to be used for the connector's TX metadata topic.
boolean	isEmitTombstoneOnDelete()
boolean	isIncrementalSnapshotSchemaChangesEnabled()
boolean	isSchemaChangesHistoryEnabled()
boolean	isSchemaCommentsHistoryEnabled()
boolean	isSignalDataCollection(DataCollectionId dataCollectionId)
private static boolean	isUsingAvroConverter(Configuration config)
Set<String>	legacyGetDataCollectionsToBeSnapshotted()	Deprecated.
Optional<String[]>	parseSignallingMessage(org.apache.kafka.connect.data.Struct value)
boolean	shouldProvideTransactionMetadata()
boolean	supportsOperationFiltering()	Whether a particular connector supports an optimized way for implementing operation skipping, or not.
protected boolean	supportsSchemaChangesDuringIncrementalSnapshot()
boolean	validate(Iterable<Field> fields, Field.ValidationOutput problems)	Validates the supplied fields in this configuration.
boolean	validateAndRecord(Iterable<Field> fields, Consumer<String> problems)	Validate the supplied fields in this configuration.
private static int	validateMaxQueueSize(Configuration config, Field field, Field.ValidationOutput problems)
static int	validateServerNameIsDifferentFromHistoryTopicName(Configuration config, Field field, Field.ValidationOutput problems)
protected static int	validateSkippedOperation(Configuration config, Field field, Field.ValidationOutput problems)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

TASK_ID

public static final String TASK_ID

See Also:

Constant Field Values

LOGGER

private static final org.slf4j.Logger LOGGER

CONFLUENT_AVRO_CONVERTER

private static final String CONFLUENT_AVRO_CONVERTER

See Also:

Constant Field Values

APICURIO_AVRO_CONVERTER

private static final String APICURIO_AVRO_CONVERTER

See Also:

Constant Field Values

DEFAULT_MAX_QUEUE_SIZE

public static final int DEFAULT_MAX_QUEUE_SIZE

See Also:

Constant Field Values

DEFAULT_MAX_BATCH_SIZE

public static final int DEFAULT_MAX_BATCH_SIZE

See Also:

Constant Field Values

DEFAULT_QUERY_FETCH_SIZE

public static final int DEFAULT_QUERY_FETCH_SIZE

See Also:

Constant Field Values

DEFAULT_POLL_INTERVAL_MILLIS

public static final long DEFAULT_POLL_INTERVAL_MILLIS

See Also:

Constant Field Values

DATABASE_CONFIG_PREFIX

public static final String DATABASE_CONFIG_PREFIX

See Also:

Constant Field Values

CONVERTER_TYPE_SUFFIX

private static final String CONVERTER_TYPE_SUFFIX

See Also:

Constant Field Values

DEFAULT_RETRIABLE_RESTART_WAIT

public static final long DEFAULT_RETRIABLE_RESTART_WAIT

See Also:

Constant Field Values

DEFAULT_MAX_QUEUE_SIZE_IN_BYTES

public static final long DEFAULT_MAX_QUEUE_SIZE_IN_BYTES

See Also:

Constant Field Values

RETRIABLE_RESTART_WAIT

public static final Field RETRIABLE_RESTART_WAIT

TOMBSTONES_ON_DELETE

public static final Field TOMBSTONES_ON_DELETE

MAX_QUEUE_SIZE

public static final Field MAX_QUEUE_SIZE

MAX_BATCH_SIZE

public static final Field MAX_BATCH_SIZE

POLL_INTERVAL_MS

public static final Field POLL_INTERVAL_MS

MAX_QUEUE_SIZE_IN_BYTES

public static final Field MAX_QUEUE_SIZE_IN_BYTES

SNAPSHOT_DELAY_MS

public static final Field SNAPSHOT_DELAY_MS

SNAPSHOT_FETCH_SIZE

public static final Field SNAPSHOT_FETCH_SIZE

INCREMENTAL_SNAPSHOT_CHUNK_SIZE

public static final Field INCREMENTAL_SNAPSHOT_CHUNK_SIZE

INCREMENTAL_SNAPSHOT_ALLOW_SCHEMA_CHANGES

public static final Field INCREMENTAL_SNAPSHOT_ALLOW_SCHEMA_CHANGES

SNAPSHOT_MODE_TABLES

public static final Field SNAPSHOT_MODE_TABLES

SOURCE_STRUCT_MAKER_VERSION

public static final Field SOURCE_STRUCT_MAKER_VERSION

SANITIZE_FIELD_NAMES

public static final Field SANITIZE_FIELD_NAMES

PROVIDE_TRANSACTION_METADATA

public static final Field PROVIDE_TRANSACTION_METADATA

EVENT_PROCESSING_FAILURE_HANDLING_MODE

public static final Field EVENT_PROCESSING_FAILURE_HANDLING_MODE

CUSTOM_CONVERTERS

public static final Field CUSTOM_CONVERTERS

SKIPPED_OPERATIONS

public static final Field SKIPPED_OPERATIONS

BINARY_HANDLING_MODE

public static final Field BINARY_HANDLING_MODE

QUERY_FETCH_SIZE

public static final Field QUERY_FETCH_SIZE

SNAPSHOT_MAX_THREADS

public static final Field SNAPSHOT_MAX_THREADS

SIGNAL_DATA_COLLECTION

public static final Field SIGNAL_DATA_COLLECTION

TRANSACTION_TOPIC

public static final Field TRANSACTION_TOPIC

CUSTOM_RETRIABLE_EXCEPTION

public static final Field CUSTOM_RETRIABLE_EXCEPTION

CONFIG_DEFINITION

protected static final ConfigDefinition CONFIG_DEFINITION

config

private final Configuration config

emitTombstoneOnDelete

private final boolean emitTombstoneOnDelete

maxQueueSize

private final int maxQueueSize

maxBatchSize

private final int maxBatchSize

maxQueueSizeInBytes

private final long maxQueueSizeInBytes

pollInterval

private final Duration pollInterval

logicalName

private final String logicalName

heartbeatTopicsPrefix

private final String heartbeatTopicsPrefix

heartbeatInterval

private final Duration heartbeatInterval

snapshotDelay

private final Duration snapshotDelay

retriableRestartWait

private final Duration retriableRestartWait

snapshotFetchSize

private final int snapshotFetchSize

incrementalSnapshotChunkSize

private final int incrementalSnapshotChunkSize

incrementalSnapshotAllowSchemaChanges

private final boolean incrementalSnapshotAllowSchemaChanges

snapshotMaxThreads

private final int snapshotMaxThreads

queryFetchSize

private final Integer queryFetchSize

sourceInfoStructMaker

private final SourceInfoStructMaker<? extends AbstractSourceInfo> sourceInfoStructMaker

sanitizeFieldNames

private final boolean sanitizeFieldNames

shouldProvideTransactionMetadata

private final boolean shouldProvideTransactionMetadata

eventProcessingFailureHandlingMode

private final CommonConnectorConfig.EventProcessingFailureHandlingMode eventProcessingFailureHandlingMode

customConverterRegistry

private final CustomConverterRegistry customConverterRegistry

binaryHandlingMode

private final CommonConnectorConfig.BinaryHandlingMode binaryHandlingMode

signalingDataCollection

private final String signalingDataCollection

skippedOperations

private final EnumSet<Envelope.Operation> skippedOperations

transactionTopic

private final String transactionTopic

taskId

private final String taskId

Constructor Detail

CommonConnectorConfig

protected CommonConnectorConfig(Configuration config,
                                String logicalName,
                                int defaultSnapshotFetchSize)

Method Detail

determineSkippedOperations

private static EnumSet<Envelope.Operation> determineSkippedOperations(Configuration config)

getConfig

@Deprecated
public Configuration getConfig()

Deprecated.

Provides access to the "raw" config instance. In most cases, access via typed getters for individual properties on the connector config class should be preferred. TODO this should be protected in the future to force proper facade methods based access / encapsulation

isEmitTombstoneOnDelete

public boolean isEmitTombstoneOnDelete()

getMaxQueueSize

public int getMaxQueueSize()

getMaxBatchSize

public int getMaxBatchSize()

getMaxQueueSizeInBytes

public long getMaxQueueSizeInBytes()

getPollInterval

public Duration getPollInterval()

getLogicalName

public String getLogicalName()

getContextName

public abstract String getContextName()

getConnectorName

public abstract String getConnectorName()

getHeartbeatTopicsPrefix

public String getHeartbeatTopicsPrefix()

getHeartbeatInterval

public Duration getHeartbeatInterval()

getRetriableRestartWait

public Duration getRetriableRestartWait()

getSnapshotDelay

public Duration getSnapshotDelay()

getSnapshotFetchSize

public int getSnapshotFetchSize()

getSnapshotMaxThreads

public int getSnapshotMaxThreads()

getQueryFetchSize

public int getQueryFetchSize()

getIncrementalSnashotChunkSize

public int getIncrementalSnashotChunkSize()

shouldProvideTransactionMetadata

public boolean shouldProvideTransactionMetadata()

getEventProcessingFailureHandlingMode

public CommonConnectorConfig.EventProcessingFailureHandlingMode getEventProcessingFailureHandlingMode()

customConverterRegistry

public CustomConverterRegistry customConverterRegistry()

getTransactionTopic

public String getTransactionTopic()

Returns the name to be used for the connector's TX metadata topic.

supportsOperationFiltering

public boolean supportsOperationFiltering()

Whether a particular connector supports an optimized way for implementing operation skipping, or not.

supportsSchemaChangesDuringIncrementalSnapshot

protected boolean supportsSchemaChangesDuringIncrementalSnapshot()

isIncrementalSnapshotSchemaChangesEnabled

public boolean isIncrementalSnapshotSchemaChangesEnabled()

getCustomConverters

private List<CustomConverter<org.apache.kafka.connect.data.SchemaBuilder,ConvertedField>> getCustomConverters()

getSourceInfoStructMaker

public <T extends AbstractSourceInfo> SourceInfoStructMaker<T> getSourceInfoStructMaker()

getSanitizeFieldNames

public boolean getSanitizeFieldNames()

getSkippedOperations

public EnumSet<Envelope.Operation> getSkippedOperations()

legacyGetDataCollectionsToBeSnapshotted

@Deprecated
public Set<String> legacyGetDataCollectionsToBeSnapshotted()

Deprecated.

getDataCollectionsToBeSnapshotted

public Set<Pattern> getDataCollectionsToBeSnapshotted()

isSchemaChangesHistoryEnabled

public boolean isSchemaChangesHistoryEnabled()

Returns:

true if the connector should emit messages about schema changes into a public facing topic.

isSchemaCommentsHistoryEnabled

public boolean isSchemaCommentsHistoryEnabled()

Returns:

true if the connector should emit messages which include table and column comments.

validate

public boolean validate(Iterable<Field> fields,
                        Field.ValidationOutput problems)

Validates the supplied fields in this configuration. Extra fields not described by the supplied fields parameter will not be validated.

Parameters:

fields - the fields

problems - the consumer to eb called with each problem; never null

Returns:

true if the value is considered valid, or false if it is not valid

validateAndRecord

public boolean validateAndRecord(Iterable<Field> fields,
                                 Consumer<String> problems)

Validate the supplied fields in this configuration. Extra fields not described by the supplied fields parameter will not be validated.

Parameters:

fields - the fields

problems - the consumer to be called with each problem; never null

Returns:

true if the value is considered valid, or false if it is not valid

validateMaxQueueSize

private static int validateMaxQueueSize(Configuration config,
                                        Field field,
                                        Field.ValidationOutput problems)

validateSkippedOperation

protected static int validateSkippedOperation(Configuration config,
                                              Field field,
                                              Field.ValidationOutput problems)

isUsingAvroConverter

private static boolean isUsingAvroConverter(Configuration config)

validateServerNameIsDifferentFromHistoryTopicName

public static int validateServerNameIsDifferentFromHistoryTopicName(Configuration config,
                                                                    Field field,
                                                                    Field.ValidationOutput problems)

getSourceInfoStructMaker

protected abstract SourceInfoStructMaker<?> getSourceInfoStructMaker(CommonConnectorConfig.Version version)

Returns the connector-specific SourceInfoStructMaker based on the given configuration.

binaryHandlingMode

public CommonConnectorConfig.BinaryHandlingMode binaryHandlingMode()

getSignalingDataCollectionId

public String getSignalingDataCollectionId()

parseSignallingMessage

public Optional<String[]> parseSignallingMessage(org.apache.kafka.connect.data.Struct value)

isSignalDataCollection

public boolean isSignalDataCollection(DataCollectionId dataCollectionId)

customRetriableException

public Optional<String> customRetriableException()

getTaskId

public String getTaskId()