Package io.debezium.pipeline.source.snapshot.incremental

Class AbstractIncrementalSnapshotContext<T>

java.lang.Object

io.debezium.pipeline.source.snapshot.incremental.AbstractIncrementalSnapshotContext<T>

All Implemented Interfaces:

IncrementalSnapshotContext<T>

Direct Known Subclasses:

SignalBasedIncrementalSnapshotContext

@NotThreadSafe
public class AbstractIncrementalSnapshotContext<T>
extends Object
implements IncrementalSnapshotContext<T>

A class describing current state of incremental snapshot

Author:

Jiri Pechanec

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
private static class	AbstractIncrementalSnapshotContext.SnapshotDataCollection<T>

Field Summary

Fields

Modifier and Type	Field	Description
private Object[]	chunkEndPosition	The last primary key in chunk that is now in process.
static final String	CORRELATION_ID
private String	correlationId
private String	currentChunkId
private final LinkedBlockingQueue<String>	dataCollectionsToStop
static final String	EVENT_PRIMARY_KEY
static final String	INCREMENTAL_SNAPSHOT_KEY
private Object[]	lastEventKeySent	The PK of the last record that was passed to Kafka Connect.
private static final org.slf4j.Logger	LOGGER
private Object[]	maximumKey	The largest PK in the table at the start of snapshot.
private final AtomicBoolean	paused	Determines if the incremental snapshot was paused or not.
private Table	schema
private boolean	schemaVerificationPassed
private final AbstractIncrementalSnapshotContext.SnapshotDataCollection<T>	snapshotDataCollection
static final String	TABLE_MAXIMUM_KEY
private final boolean	useCatalogBeforeSchema
protected boolean	windowOpened	true if window is opened and deduplication should be executed

Constructor Summary

Constructors

Constructor	Description
AbstractIncrementalSnapshotContext(boolean useCatalogBeforeSchema)

Method Summary

Modifier and Type	Method	Description
List<DataCollection<T>>	addDataCollectionNamesToSnapshot(String correlationId, List<String> dataCollectionIds, List<AdditionalCondition> additionalCondition, String surrogateKey)
private void	addTablesIdsToSnapshot(List<DataCollection<T>> dataCollectionIds)
private String	arrayToSerializedString(Object[] array)
private Function<String,Optional<DataCollection<T>>>	buildDataCollection(List<AdditionalCondition> additionalCondition, String surrogateKey)
Object[]	chunkEndPosititon()
boolean	closeWindow(String id)
String	currentChunkId()
DataCollection<T>	currentDataCollectionId()
int	dataCollectionsToBeSnapshottedCount()
boolean	deduplicationNeeded()
String	getCorrelationId()
List<DataCollection<T>>	getDataCollections()
List<String>	getDataCollectionsToStop()
Table	getSchema()
protected static <U> IncrementalSnapshotContext<U>	init(AbstractIncrementalSnapshotContext<U> context, Map<String,?> offsets)
boolean	isNonInitialChunk()
boolean	isSchemaVerificationPassed()
boolean	isSnapshotPaused()
Optional<Object[]>	maximumKey()
void	maximumKey(Object[] key)
void	nextChunkPosition(Object[] end)
DataCollection<T>	nextDataCollection()
private boolean	notExpectedChunk(String id)	The snapshotting process can receive out-of-order windowing signals after connector restart as depending on committed offset position some signals can be replayed.
boolean	openWindow(String id)
void	pauseSnapshot()
boolean	removeDataCollectionFromSnapshot(String dataCollectionId)
void	requestSnapshotStop(List<String> dataCollectionIds)
private void	resetChunk()
void	resumeSnapshot()
void	revertChunk()
void	sendEvent(Object[] key)
private Object[]	serializedStringToArray(String field, String serialized)
void	setSchema(Table schema)
void	setSchemaVerificationPassed(boolean schemaVerificationPassed)
boolean	snapshotRunning()
void	startNewChunk()
Map<String,Object>	store(Map<String,Object> offset)
String	toString()
void	unsetCorrelationId()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait

Field Details

LOGGER

private static final org.slf4j.Logger LOGGER

INCREMENTAL_SNAPSHOT_KEY

public static final String INCREMENTAL_SNAPSHOT_KEY

See Also:

• Constant Field Values

EVENT_PRIMARY_KEY

public static final String EVENT_PRIMARY_KEY

See Also:

• Constant Field Values

TABLE_MAXIMUM_KEY

public static final String TABLE_MAXIMUM_KEY

See Also:

• Constant Field Values

CORRELATION_ID

public static final String CORRELATION_ID

See Also:

• Constant Field Values

snapshotDataCollection

private final AbstractIncrementalSnapshotContext.SnapshotDataCollection<T> snapshotDataCollection

windowOpened

protected boolean windowOpened

true if window is opened and deduplication should be executed

chunkEndPosition

private Object[] chunkEndPosition

The last primary key in chunk that is now in process.

useCatalogBeforeSchema

private final boolean useCatalogBeforeSchema

lastEventKeySent

private Object[] lastEventKeySent

The PK of the last record that was passed to Kafka Connect. In case of connector restart the start of the first chunk will be populated from it.

currentChunkId

private String currentChunkId

maximumKey

private Object[] maximumKey

The largest PK in the table at the start of snapshot.

schema

private Table schema

schemaVerificationPassed

private boolean schemaVerificationPassed

correlationId

private String correlationId

paused

private final AtomicBoolean paused

Determines if the incremental snapshot was paused or not.

dataCollectionsToStop

private final LinkedBlockingQueue<String> dataCollectionsToStop

Constructor Details

AbstractIncrementalSnapshotContext

public AbstractIncrementalSnapshotContext(boolean useCatalogBeforeSchema)

Method Details

openWindow

public boolean openWindow(String id)

Specified by:

openWindow in interface IncrementalSnapshotContext<T>

closeWindow

public boolean closeWindow(String id)

Specified by:

closeWindow in interface IncrementalSnapshotContext<T>

pauseSnapshot

public void pauseSnapshot()

Specified by:

pauseSnapshot in interface IncrementalSnapshotContext<T>

resumeSnapshot

public void resumeSnapshot()

Specified by:

resumeSnapshot in interface IncrementalSnapshotContext<T>

isSnapshotPaused

public boolean isSnapshotPaused()

Specified by:

isSnapshotPaused in interface IncrementalSnapshotContext<T>

notExpectedChunk

private boolean notExpectedChunk(String id)

The snapshotting process can receive out-of-order windowing signals after connector restart as depending on committed offset position some signals can be replayed. In extreme case a signal can be received even when the incremental snapshot was completed just before the restart. Such windowing signals are ignored.

deduplicationNeeded

public boolean deduplicationNeeded()

Specified by:

deduplicationNeeded in interface IncrementalSnapshotContext<T>

arrayToSerializedString

private String arrayToSerializedString(Object[] array)

serializedStringToArray

private Object[] serializedStringToArray(String field,
 String serialized)

getDataCollectionsToStop

public List<String> getDataCollectionsToStop()

Specified by:

getDataCollectionsToStop in interface IncrementalSnapshotContext<T>

snapshotRunning

public boolean snapshotRunning()

Specified by:

snapshotRunning in interface IncrementalSnapshotContext<T>

store

public Map<String,Object> store(Map<String,Object> offset)

Specified by:

store in interface IncrementalSnapshotContext<T>

addTablesIdsToSnapshot

private void addTablesIdsToSnapshot(List<DataCollection<T>> dataCollectionIds)

addDataCollectionNamesToSnapshot

public List<DataCollection<T>> addDataCollectionNamesToSnapshot(String correlationId,
 List<String> dataCollectionIds,
 List<AdditionalCondition> additionalCondition,
 String surrogateKey)

Specified by:

addDataCollectionNamesToSnapshot in interface IncrementalSnapshotContext<T>

buildDataCollection

private Function<String,Optional<DataCollection<T>>> buildDataCollection(List<AdditionalCondition> additionalCondition,
 String surrogateKey)

requestSnapshotStop

public void requestSnapshotStop(List<String> dataCollectionIds)

Specified by:

requestSnapshotStop in interface IncrementalSnapshotContext<T>

removeDataCollectionFromSnapshot

public boolean removeDataCollectionFromSnapshot(String dataCollectionId)

Specified by:

removeDataCollectionFromSnapshot in interface IncrementalSnapshotContext<T>

getDataCollections

public List<DataCollection<T>> getDataCollections()

Specified by:

getDataCollections in interface IncrementalSnapshotContext<T>

unsetCorrelationId

public void unsetCorrelationId()

Specified by:

unsetCorrelationId in interface IncrementalSnapshotContext<T>

getCorrelationId

public String getCorrelationId()

Specified by:

getCorrelationId in interface IncrementalSnapshotContext<T>

init

protected static <U> IncrementalSnapshotContext<U> init(AbstractIncrementalSnapshotContext<U> context,
 Map<String,?> offsets)

sendEvent

public void sendEvent(Object[] key)

Specified by:

sendEvent in interface IncrementalSnapshotContext<T>

currentDataCollectionId

public DataCollection<T> currentDataCollectionId()

Specified by:

currentDataCollectionId in interface IncrementalSnapshotContext<T>

dataCollectionsToBeSnapshottedCount

public int dataCollectionsToBeSnapshottedCount()

Specified by:

dataCollectionsToBeSnapshottedCount in interface IncrementalSnapshotContext<T>

nextChunkPosition

public void nextChunkPosition(Object[] end)

Specified by:

nextChunkPosition in interface IncrementalSnapshotContext<T>

chunkEndPosititon

public Object[] chunkEndPosititon()

Specified by:

chunkEndPosititon in interface IncrementalSnapshotContext<T>

resetChunk

private void resetChunk()

revertChunk

public void revertChunk()

Specified by:

revertChunk in interface IncrementalSnapshotContext<T>

isNonInitialChunk

public boolean isNonInitialChunk()

Specified by:

isNonInitialChunk in interface IncrementalSnapshotContext<T>

nextDataCollection

public DataCollection<T> nextDataCollection()

Specified by:

nextDataCollection in interface IncrementalSnapshotContext<T>

startNewChunk

public void startNewChunk()

Specified by:

startNewChunk in interface IncrementalSnapshotContext<T>

currentChunkId

public String currentChunkId()

Specified by:

currentChunkId in interface IncrementalSnapshotContext<T>

maximumKey

public void maximumKey(Object[] key)

Specified by:

maximumKey in interface IncrementalSnapshotContext<T>

maximumKey

public Optional<Object[]> maximumKey()

Specified by:

maximumKey in interface IncrementalSnapshotContext<T>

getSchema

public Table getSchema()

Specified by:

getSchema in interface IncrementalSnapshotContext<T>

setSchema

public void setSchema(Table schema)

Specified by:

setSchema in interface IncrementalSnapshotContext<T>

isSchemaVerificationPassed

public boolean isSchemaVerificationPassed()

Specified by:

isSchemaVerificationPassed in interface IncrementalSnapshotContext<T>

setSchemaVerificationPassed

public void setSchemaVerificationPassed(boolean schemaVerificationPassed)

Specified by:

setSchemaVerificationPassed in interface IncrementalSnapshotContext<T>

toString

public String toString()

Overrides:

toString in class Object