Package io.debezium.pipeline.source.snapshot.incremental

Class AbstractIncrementalSnapshotContext<T>

java.lang.Object

io.debezium.pipeline.source.snapshot.incremental.AbstractIncrementalSnapshotContext<T>

All Implemented Interfaces:

IncrementalSnapshotContext<T>

Direct Known Subclasses:

SignalBasedIncrementalSnapshotContext

@NotThreadSafe
public class AbstractIncrementalSnapshotContext<T>
extends Object
implements IncrementalSnapshotContext<T>

A class describing current state of incremental snapshot

Author:

Jiri Pechanec

Field Summary

Fields

Modifier and Type	Field	Description
private Object[]	chunkEndPosition	The last primary key in chunk that is now in process.
private String	currentChunkId
static String	DATA_COLLECTIONS_TO_SNAPSHOT_KEY
private Queue<T>	dataCollectionsToSnapshot
static String	EVENT_PRIMARY_KEY
static String	INCREMENTAL_SNAPSHOT_KEY
private Object[]	lastEventKeySent	The PK of the last record that was passed to Kafka Connect.
private static org.slf4j.Logger	LOGGER
private Object[]	maximumKey	The largest PK in the table at the start of snapshot.
private Table	schema
private boolean	schemaVerificationPassed
static String	TABLE_MAXIMUM_KEY
private boolean	useCatalogBeforeSchema
protected boolean	windowOpened

Constructor Summary

Constructors

Constructor	Description
AbstractIncrementalSnapshotContext(boolean useCatalogBeforeSchema)

Method Summary

Modifier and Type	Method	Description
List<T>	addDataCollectionNamesToSnapshot(List<String> dataCollectionIds)
private void	addTablesIdsToSnapshot(List<T> dataCollectionIds)
private String	arrayToSerializedString(Object[] array)
Object[]	chunkEndPosititon()
boolean	closeWindow(String id)
String	currentChunkId()
T	currentDataCollectionId()
int	dataCollectionsToBeSnapshottedCount()
private String	dataCollectionsToSnapshotAsString()
boolean	deduplicationNeeded()
Table	getSchema()
protected static <U> IncrementalSnapshotContext<U>	init(AbstractIncrementalSnapshotContext<U> context, Map<String,?> offsets)
boolean	isNonInitialChunk()
boolean	isSchemaVerificationPassed()
Optional<Object[]>	maximumKey()
void	maximumKey(Object[] key)
void	nextChunkPosition(Object[] end)
T	nextDataCollection()
private boolean	notExpectedChunk(String id)	The snapshotting process can receive out-of-order windowing signals after connector restart as depending on committed offset position some signals can be replayed.
boolean	openWindow(String id)
private void	resetChunk()
void	revertChunk()
void	sendEvent(Object[] key)
private Object[]	serializedStringToArray(String field, String serialized)
void	setSchema(Table schema)
void	setSchemaVerificationPassed(boolean schemaVerificationPassed)
boolean	snapshotRunning()
void	startNewChunk()
Map<String,Object>	store(Map<String,Object> offset)
private List<String>	stringToDataCollections(String dataCollectionsStr)
String	toString()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait

Field Detail

LOGGER

private static final org.slf4j.Logger LOGGER

INCREMENTAL_SNAPSHOT_KEY

public static final String INCREMENTAL_SNAPSHOT_KEY

See Also:

Constant Field Values

DATA_COLLECTIONS_TO_SNAPSHOT_KEY

public static final String DATA_COLLECTIONS_TO_SNAPSHOT_KEY

See Also:

Constant Field Values

EVENT_PRIMARY_KEY

public static final String EVENT_PRIMARY_KEY

See Also:

Constant Field Values

TABLE_MAXIMUM_KEY

public static final String TABLE_MAXIMUM_KEY

See Also:

Constant Field Values

windowOpened

protected boolean windowOpened

chunkEndPosition

private Object[] chunkEndPosition

The last primary key in chunk that is now in process.

dataCollectionsToSnapshot

private final Queue<T> dataCollectionsToSnapshot

useCatalogBeforeSchema

private final boolean useCatalogBeforeSchema

lastEventKeySent

private Object[] lastEventKeySent

The PK of the last record that was passed to Kafka Connect. In case of connector restart the start of the first chunk will be populated from it.

currentChunkId

private String currentChunkId

maximumKey

private Object[] maximumKey

The largest PK in the table at the start of snapshot.

schema

private Table schema

schemaVerificationPassed

private boolean schemaVerificationPassed

Constructor Detail

AbstractIncrementalSnapshotContext

public AbstractIncrementalSnapshotContext(boolean useCatalogBeforeSchema)

Method Detail

openWindow

public boolean openWindow(String id)

Specified by:

openWindow in interface IncrementalSnapshotContext<T>

closeWindow

public boolean closeWindow(String id)

Specified by:

closeWindow in interface IncrementalSnapshotContext<T>

notExpectedChunk

private boolean notExpectedChunk(String id)

The snapshotting process can receive out-of-order windowing signals after connector restart as depending on committed offset position some signals can be replayed. In extreme case a signal can be received even when the incremental snapshot was completed just before the restart. Such windowing signals are ignored.

deduplicationNeeded

public boolean deduplicationNeeded()

Specified by:

deduplicationNeeded in interface IncrementalSnapshotContext<T>

arrayToSerializedString

private String arrayToSerializedString(Object[] array)

serializedStringToArray

private Object[] serializedStringToArray(String field,
                                         String serialized)

dataCollectionsToSnapshotAsString

private String dataCollectionsToSnapshotAsString()

stringToDataCollections

private List<String> stringToDataCollections(String dataCollectionsStr)

snapshotRunning

public boolean snapshotRunning()

Specified by:

snapshotRunning in interface IncrementalSnapshotContext<T>

store

public Map<String,Object> store(Map<String,Object> offset)

Specified by:

store in interface IncrementalSnapshotContext<T>

addTablesIdsToSnapshot

private void addTablesIdsToSnapshot(List<T> dataCollectionIds)

addDataCollectionNamesToSnapshot

public List<T> addDataCollectionNamesToSnapshot(List<String> dataCollectionIds)

Specified by:

addDataCollectionNamesToSnapshot in interface IncrementalSnapshotContext<T>

init

protected static <U> IncrementalSnapshotContext<U> init(AbstractIncrementalSnapshotContext<U> context,
                                                        Map<String,?> offsets)

sendEvent

public void sendEvent(Object[] key)

Specified by:

sendEvent in interface IncrementalSnapshotContext<T>

currentDataCollectionId

public T currentDataCollectionId()

Specified by:

currentDataCollectionId in interface IncrementalSnapshotContext<T>

dataCollectionsToBeSnapshottedCount

public int dataCollectionsToBeSnapshottedCount()

Specified by:

dataCollectionsToBeSnapshottedCount in interface IncrementalSnapshotContext<T>

nextChunkPosition

public void nextChunkPosition(Object[] end)

Specified by:

nextChunkPosition in interface IncrementalSnapshotContext<T>

chunkEndPosititon

public Object[] chunkEndPosititon()

Specified by:

chunkEndPosititon in interface IncrementalSnapshotContext<T>

resetChunk

private void resetChunk()

revertChunk

public void revertChunk()

Specified by:

revertChunk in interface IncrementalSnapshotContext<T>

isNonInitialChunk

public boolean isNonInitialChunk()

Specified by:

isNonInitialChunk in interface IncrementalSnapshotContext<T>

nextDataCollection

public T nextDataCollection()

Specified by:

nextDataCollection in interface IncrementalSnapshotContext<T>

startNewChunk

public void startNewChunk()

Specified by:

startNewChunk in interface IncrementalSnapshotContext<T>

currentChunkId

public String currentChunkId()

Specified by:

currentChunkId in interface IncrementalSnapshotContext<T>

maximumKey

public void maximumKey(Object[] key)

Specified by:

maximumKey in interface IncrementalSnapshotContext<T>

maximumKey

public Optional<Object[]> maximumKey()

Specified by:

maximumKey in interface IncrementalSnapshotContext<T>

getSchema

public Table getSchema()

Specified by:

getSchema in interface IncrementalSnapshotContext<T>

setSchema

public void setSchema(Table schema)

Specified by:

setSchema in interface IncrementalSnapshotContext<T>

isSchemaVerificationPassed

public boolean isSchemaVerificationPassed()

Specified by:

isSchemaVerificationPassed in interface IncrementalSnapshotContext<T>

setSchemaVerificationPassed

public void setSchemaVerificationPassed(boolean schemaVerificationPassed)

Specified by:

setSchemaVerificationPassed in interface IncrementalSnapshotContext<T>

toString

public String toString()

Overrides:

toString in class Object