Package io.debezium.pipeline.source.snapshot.incremental

Class AbstractIncrementalSnapshotContext<T>

java.lang.Object

io.debezium.pipeline.source.snapshot.incremental.AbstractIncrementalSnapshotContext<T>

All Implemented Interfaces:

IncrementalSnapshotContext<T>

Direct Known Subclasses:

SignalBasedIncrementalSnapshotContext

@NotThreadSafe
public class AbstractIncrementalSnapshotContext<T>
extends Object
implements IncrementalSnapshotContext<T>

A class describing current state of incremental snapshot

Author:

Jiri Pechanec

Field Summary

Fields

Modifier and Type	Field	Description
private Object[]	chunkEndPosition	The last primary key in chunk that is now in process.
private String	currentChunkId
static final String	DATA_COLLECTIONS_TO_SNAPSHOT_KEY
static final String	DATA_COLLECTIONS_TO_SNAPSHOT_KEY_ADDITIONAL_CONDITION
static final String	DATA_COLLECTIONS_TO_SNAPSHOT_KEY_ID
private final Queue<DataCollection<T>>	dataCollectionsToSnapshot
static final String	EVENT_PRIMARY_KEY
static final String	INCREMENTAL_SNAPSHOT_KEY
private Object[]	lastEventKeySent	The PK of the last record that was passed to Kafka Connect.
private static final org.slf4j.Logger	LOGGER
private com.fasterxml.jackson.databind.ObjectMapper	mapper
private com.fasterxml.jackson.core.type.TypeReference<List<LinkedHashMap<String,String>>>	mapperTypeRef
private Object[]	maximumKey	The largest PK in the table at the start of snapshot.
private AtomicBoolean	paused	Determines if the incremental snapshot was paused or not.
private Table	schema
private boolean	schemaVerificationPassed
static final String	TABLE_MAXIMUM_KEY
private final boolean	useCatalogBeforeSchema
protected boolean	windowOpened

Constructor Summary

Constructors

Constructor	Description
AbstractIncrementalSnapshotContext(boolean useCatalogBeforeSchema)

Method Summary

Modifier and Type	Method	Description
List<DataCollection<T>>	addDataCollectionNamesToSnapshot(List<String> dataCollectionIds, Optional<String> additionalCondition)
private void	addTablesIdsToSnapshot(List<DataCollection<T>> dataCollectionIds)
private String	arrayToSerializedString(Object[] array)
Object[]	chunkEndPosititon()
boolean	closeWindow(String id)
String	currentChunkId()
DataCollection<T>	currentDataCollectionId()
int	dataCollectionsToBeSnapshottedCount()
private String	dataCollectionsToSnapshotAsString()
boolean	deduplicationNeeded()
Table	getSchema()
protected static <U> IncrementalSnapshotContext<U>	init(AbstractIncrementalSnapshotContext<U> context, Map<String,?> offsets)
boolean	isNonInitialChunk()
boolean	isSchemaVerificationPassed()
boolean	isSnapshotPaused()
Optional<Object[]>	maximumKey()
void	maximumKey(Object[] key)
void	nextChunkPosition(Object[] end)
DataCollection<T>	nextDataCollection()
private boolean	notExpectedChunk(String id)	The snapshotting process can receive out-of-order windowing signals after connector restart as depending on committed offset position some signals can be replayed.
boolean	openWindow(String id)
void	pauseSnapshot()
boolean	removeDataCollectionFromSnapshot(String dataCollectionId)
private void	resetChunk()
void	resumeSnapshot()
void	revertChunk()
void	sendEvent(Object[] key)
private Object[]	serializedStringToArray(String field, String serialized)
void	setSchema(Table schema)
void	setSchemaVerificationPassed(boolean schemaVerificationPassed)
boolean	snapshotRunning()
void	startNewChunk()
void	stopSnapshot()
Map<String,Object>	store(Map<String,Object> offset)
private List<DataCollection<T>>	stringToDataCollections(String dataCollectionsStr)
String	toString()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait

Field Details

LOGGER

private static final org.slf4j.Logger LOGGER

INCREMENTAL_SNAPSHOT_KEY

public static final String INCREMENTAL_SNAPSHOT_KEY

See Also:

• Constant Field Values

DATA_COLLECTIONS_TO_SNAPSHOT_KEY

public static final String DATA_COLLECTIONS_TO_SNAPSHOT_KEY

See Also:

• Constant Field Values

DATA_COLLECTIONS_TO_SNAPSHOT_KEY_ID

public static final String DATA_COLLECTIONS_TO_SNAPSHOT_KEY_ID

See Also:

• Constant Field Values

DATA_COLLECTIONS_TO_SNAPSHOT_KEY_ADDITIONAL_CONDITION

public static final String DATA_COLLECTIONS_TO_SNAPSHOT_KEY_ADDITIONAL_CONDITION

See Also:

• Constant Field Values

EVENT_PRIMARY_KEY

public static final String EVENT_PRIMARY_KEY

See Also:

• Constant Field Values

TABLE_MAXIMUM_KEY

public static final String TABLE_MAXIMUM_KEY

See Also:

• Constant Field Values

windowOpened

protected boolean windowOpened

chunkEndPosition

private Object[] chunkEndPosition

The last primary key in chunk that is now in process.

dataCollectionsToSnapshot

private final Queue<DataCollection<T>> dataCollectionsToSnapshot

useCatalogBeforeSchema

private final boolean useCatalogBeforeSchema

lastEventKeySent

private Object[] lastEventKeySent

The PK of the last record that was passed to Kafka Connect. In case of connector restart the start of the first chunk will be populated from it.

currentChunkId

private String currentChunkId

maximumKey

private Object[] maximumKey

The largest PK in the table at the start of snapshot.

schema

private Table schema

schemaVerificationPassed

private boolean schemaVerificationPassed

paused

private AtomicBoolean paused

Determines if the incremental snapshot was paused or not.

mapper

private com.fasterxml.jackson.databind.ObjectMapper mapper

mapperTypeRef

private com.fasterxml.jackson.core.type.TypeReference<List<LinkedHashMap<String,String>>> mapperTypeRef

Constructor Details

AbstractIncrementalSnapshotContext

public AbstractIncrementalSnapshotContext(boolean useCatalogBeforeSchema)

Method Details

openWindow

public boolean openWindow(String id)

Specified by:

openWindow in interface IncrementalSnapshotContext<T>

closeWindow

public boolean closeWindow(String id)

Specified by:

closeWindow in interface IncrementalSnapshotContext<T>

pauseSnapshot

public void pauseSnapshot()

Specified by:

pauseSnapshot in interface IncrementalSnapshotContext<T>

resumeSnapshot

public void resumeSnapshot()

Specified by:

resumeSnapshot in interface IncrementalSnapshotContext<T>

isSnapshotPaused

public boolean isSnapshotPaused()

Specified by:

isSnapshotPaused in interface IncrementalSnapshotContext<T>

notExpectedChunk

private boolean notExpectedChunk(String id)

The snapshotting process can receive out-of-order windowing signals after connector restart as depending on committed offset position some signals can be replayed. In extreme case a signal can be received even when the incremental snapshot was completed just before the restart. Such windowing signals are ignored.

deduplicationNeeded

public boolean deduplicationNeeded()

Specified by:

deduplicationNeeded in interface IncrementalSnapshotContext<T>

arrayToSerializedString

private String arrayToSerializedString(Object[] array)

serializedStringToArray

private Object[] serializedStringToArray(String field,
 String serialized)

dataCollectionsToSnapshotAsString

private String dataCollectionsToSnapshotAsString()

stringToDataCollections

private List<DataCollection<T>> stringToDataCollections(String dataCollectionsStr)

snapshotRunning

public boolean snapshotRunning()

Specified by:

snapshotRunning in interface IncrementalSnapshotContext<T>

store

public Map<String,Object> store(Map<String,Object> offset)

Specified by:

store in interface IncrementalSnapshotContext<T>

addTablesIdsToSnapshot

private void addTablesIdsToSnapshot(List<DataCollection<T>> dataCollectionIds)

addDataCollectionNamesToSnapshot

public List<DataCollection<T>> addDataCollectionNamesToSnapshot(List<String> dataCollectionIds,
 Optional<String> additionalCondition)

Specified by:

addDataCollectionNamesToSnapshot in interface IncrementalSnapshotContext<T>

stopSnapshot

public void stopSnapshot()

Specified by:

stopSnapshot in interface IncrementalSnapshotContext<T>

removeDataCollectionFromSnapshot

public boolean removeDataCollectionFromSnapshot(String dataCollectionId)

Specified by:

removeDataCollectionFromSnapshot in interface IncrementalSnapshotContext<T>

init

protected static <U> IncrementalSnapshotContext<U> init(AbstractIncrementalSnapshotContext<U> context,
 Map<String,?> offsets)

sendEvent

public void sendEvent(Object[] key)

Specified by:

sendEvent in interface IncrementalSnapshotContext<T>

currentDataCollectionId

public DataCollection<T> currentDataCollectionId()

Specified by:

currentDataCollectionId in interface IncrementalSnapshotContext<T>

dataCollectionsToBeSnapshottedCount

public int dataCollectionsToBeSnapshottedCount()

Specified by:

dataCollectionsToBeSnapshottedCount in interface IncrementalSnapshotContext<T>

nextChunkPosition

public void nextChunkPosition(Object[] end)

Specified by:

nextChunkPosition in interface IncrementalSnapshotContext<T>

chunkEndPosititon

public Object[] chunkEndPosititon()

Specified by:

chunkEndPosititon in interface IncrementalSnapshotContext<T>

resetChunk

private void resetChunk()

revertChunk

public void revertChunk()

Specified by:

revertChunk in interface IncrementalSnapshotContext<T>

isNonInitialChunk

public boolean isNonInitialChunk()

Specified by:

isNonInitialChunk in interface IncrementalSnapshotContext<T>

nextDataCollection

public DataCollection<T> nextDataCollection()

Specified by:

nextDataCollection in interface IncrementalSnapshotContext<T>

startNewChunk

public void startNewChunk()

Specified by:

startNewChunk in interface IncrementalSnapshotContext<T>

currentChunkId

public String currentChunkId()

Specified by:

currentChunkId in interface IncrementalSnapshotContext<T>

maximumKey

public void maximumKey(Object[] key)

Specified by:

maximumKey in interface IncrementalSnapshotContext<T>

maximumKey

public Optional<Object[]> maximumKey()

Specified by:

maximumKey in interface IncrementalSnapshotContext<T>

getSchema

public Table getSchema()

Specified by:

getSchema in interface IncrementalSnapshotContext<T>

setSchema

public void setSchema(Table schema)

Specified by:

setSchema in interface IncrementalSnapshotContext<T>

isSchemaVerificationPassed

public boolean isSchemaVerificationPassed()

Specified by:

isSchemaVerificationPassed in interface IncrementalSnapshotContext<T>

setSchemaVerificationPassed

public void setSchemaVerificationPassed(boolean schemaVerificationPassed)

Specified by:

setSchemaVerificationPassed in interface IncrementalSnapshotContext<T>

toString

public String toString()

Overrides:

toString in class Object