Class AbstractIncrementalSnapshotChangeEventSource<T extends DataCollectionId>
- java.lang.Object
-
- io.debezium.pipeline.source.snapshot.incremental.AbstractIncrementalSnapshotChangeEventSource<T>
-
- All Implemented Interfaces:
IncrementalSnapshotChangeEventSource<T>
- Direct Known Subclasses:
SignalBasedIncrementalSnapshotChangeEventSource
@NotThreadSafe public abstract class AbstractIncrementalSnapshotChangeEventSource<T extends DataCollectionId> extends Object implements IncrementalSnapshotChangeEventSource<T>
An incremental snapshot change event source that emits events from a DB log interleaved with snapshot events.
-
-
Field Summary
Fields Modifier and Type Field Description private Clock
clock
private CommonConnectorConfig
connectorConfig
protected IncrementalSnapshotContext<T>
context
private Table
currentTable
private RelationalDatabaseSchema
databaseSchema
private DataChangeEventListener
dataListener
protected EventDispatcher<T>
dispatcher
protected JdbcConnection
jdbcConnection
private static org.slf4j.Logger
LOGGER
private SnapshotProgressListener
progressListener
private long
totalRowsScanned
protected Map<org.apache.kafka.connect.data.Struct,Object[]>
window
-
Constructor Summary
Constructors Constructor Description AbstractIncrementalSnapshotChangeEventSource(CommonConnectorConfig config, JdbcConnection jdbcConnection, EventDispatcher<T> dispatcher, DatabaseSchema<?> databaseSchema, Clock clock, SnapshotProgressListener progressListener, DataChangeEventListener dataChangeEventListener)
-
Method Summary
All Methods Instance Methods Abstract Methods Concrete Methods Modifier and Type Method Description void
addDataCollectionNamesToSnapshot(List<String> dataCollectionIds, OffsetContext offsetContext)
protected void
addKeyColumnsToCondition(Table table, StringBuilder sql, String predicate)
private void
addLowerBound(Table table, StringBuilder sql)
protected String
buildChunkQuery(Table table)
protected String
buildMaxPrimaryKeyQuery(Table table)
void
closeWindow(Partition partition, String id, OffsetContext offsetContext)
private void
createDataEventsForTable()
Dispatches the data change events for the records of a single table.protected void
deduplicateWindow(DataCollectionId dataCollectionId, Object key)
protected abstract void
emitWindowClose()
Update high watermark for the incremental snapshot chunkprotected abstract void
emitWindowOpen()
Update low watermark for the incremental snapshot chunkprotected ChangeRecordEmitter
getChangeRecordEmitter(Partition partition, T dataCollectionId, OffsetContext offsetContext, Object[] row)
Returns aChangeRecordEmitter
producing the change records for the given table row.protected String
getSignalTableName(String dataCollectionId)
private Threads.Timer
getTableScanLogTimer()
private void
incrementTableRowsScanned(long rows)
void
init(OffsetContext offsetContext)
private Object[]
keyFromRow(Object[] row)
private void
nextDataCollection()
protected void
postIncrementalSnapshotCompleted()
protected void
postReadChunk(IncrementalSnapshotContext<T> context)
protected void
preReadChunk(IncrementalSnapshotContext<T> context)
protected void
readChunk()
protected PreparedStatement
readTableChunkStatement(String sql)
protected void
sendEvent(Partition partition, EventDispatcher<T> dispatcher, OffsetContext offsetContext, Object[] row)
protected void
sendWindowEvents(Partition partition, OffsetContext offsetContext)
protected void
setContext(IncrementalSnapshotContext<T> context)
private void
tableScanCompleted()
-
Methods inherited from class java.lang.Object
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
-
Methods inherited from interface io.debezium.pipeline.source.snapshot.incremental.IncrementalSnapshotChangeEventSource
processFilteredEvent, processHeartbeat, processMessage, processTransactionCommittedEvent, processTransactionStartedEvent
-
-
-
-
Field Detail
-
LOGGER
private static final org.slf4j.Logger LOGGER
-
connectorConfig
private final CommonConnectorConfig connectorConfig
-
clock
private final Clock clock
-
databaseSchema
private final RelationalDatabaseSchema databaseSchema
-
progressListener
private final SnapshotProgressListener progressListener
-
dataListener
private final DataChangeEventListener dataListener
-
totalRowsScanned
private long totalRowsScanned
-
currentTable
private Table currentTable
-
dispatcher
protected EventDispatcher<T extends DataCollectionId> dispatcher
-
context
protected IncrementalSnapshotContext<T extends DataCollectionId> context
-
jdbcConnection
protected JdbcConnection jdbcConnection
-
-
Constructor Detail
-
AbstractIncrementalSnapshotChangeEventSource
public AbstractIncrementalSnapshotChangeEventSource(CommonConnectorConfig config, JdbcConnection jdbcConnection, EventDispatcher<T> dispatcher, DatabaseSchema<?> databaseSchema, Clock clock, SnapshotProgressListener progressListener, DataChangeEventListener dataChangeEventListener)
-
-
Method Detail
-
closeWindow
public void closeWindow(Partition partition, String id, OffsetContext offsetContext) throws InterruptedException
- Specified by:
closeWindow
in interfaceIncrementalSnapshotChangeEventSource<T extends DataCollectionId>
- Throws:
InterruptedException
-
sendWindowEvents
protected void sendWindowEvents(Partition partition, OffsetContext offsetContext) throws InterruptedException
- Throws:
InterruptedException
-
sendEvent
protected void sendEvent(Partition partition, EventDispatcher<T> dispatcher, OffsetContext offsetContext, Object[] row) throws InterruptedException
- Throws:
InterruptedException
-
getChangeRecordEmitter
protected ChangeRecordEmitter getChangeRecordEmitter(Partition partition, T dataCollectionId, OffsetContext offsetContext, Object[] row)
Returns aChangeRecordEmitter
producing the change records for the given table row.
-
deduplicateWindow
protected void deduplicateWindow(DataCollectionId dataCollectionId, Object key)
-
emitWindowOpen
protected abstract void emitWindowOpen() throws SQLException
Update low watermark for the incremental snapshot chunk- Throws:
SQLException
-
emitWindowClose
protected abstract void emitWindowClose() throws SQLException, InterruptedException
Update high watermark for the incremental snapshot chunk- Throws:
SQLException
InterruptedException
-
addLowerBound
private void addLowerBound(Table table, StringBuilder sql)
-
init
public void init(OffsetContext offsetContext)
- Specified by:
init
in interfaceIncrementalSnapshotChangeEventSource<T extends DataCollectionId>
-
readChunk
protected void readChunk() throws InterruptedException
- Throws:
InterruptedException
-
nextDataCollection
private void nextDataCollection()
-
addDataCollectionNamesToSnapshot
public void addDataCollectionNamesToSnapshot(List<String> dataCollectionIds, OffsetContext offsetContext) throws InterruptedException
- Specified by:
addDataCollectionNamesToSnapshot
in interfaceIncrementalSnapshotChangeEventSource<T extends DataCollectionId>
- Throws:
InterruptedException
-
addKeyColumnsToCondition
protected void addKeyColumnsToCondition(Table table, StringBuilder sql, String predicate)
-
createDataEventsForTable
private void createDataEventsForTable()
Dispatches the data change events for the records of a single table.
-
incrementTableRowsScanned
private void incrementTableRowsScanned(long rows)
-
tableScanCompleted
private void tableScanCompleted()
-
readTableChunkStatement
protected PreparedStatement readTableChunkStatement(String sql) throws SQLException
- Throws:
SQLException
-
getTableScanLogTimer
private Threads.Timer getTableScanLogTimer()
-
setContext
protected void setContext(IncrementalSnapshotContext<T> context)
-
preReadChunk
protected void preReadChunk(IncrementalSnapshotContext<T> context)
-
postReadChunk
protected void postReadChunk(IncrementalSnapshotContext<T> context)
-
postIncrementalSnapshotCompleted
protected void postIncrementalSnapshotCompleted()
-
-