public class PubsubUnboundedSink extends org.apache.beam.sdk.transforms.PTransform<org.apache.beam.sdk.values.PCollection<PubsubMessage>,org.apache.beam.sdk.values.PDone>
GroupByKey
followed by a ParDo
which publishes as a side effect. (In the future we want to design and switch to a custom
UnboundedSink
implementation so as to gain access to system watermark and
end-of-pipeline cleanup.)
DoFn
instances to execute concurrently and hide latency.
Constructor and Description |
---|
PubsubUnboundedSink(PubsubClient.PubsubClientFactory pubsubFactory,
org.apache.beam.sdk.options.ValueProvider<PubsubClient.TopicPath> topic,
java.lang.String timestampAttribute,
java.lang.String idAttribute,
int numShards) |
PubsubUnboundedSink(PubsubClient.PubsubClientFactory pubsubFactory,
org.apache.beam.sdk.options.ValueProvider<PubsubClient.TopicPath> topic,
java.lang.String timestampAttribute,
java.lang.String idAttribute,
int numShards,
int publishBatchSize,
int publishBatchBytes) |
Modifier and Type | Method and Description |
---|---|
org.apache.beam.sdk.values.PDone |
expand(org.apache.beam.sdk.values.PCollection<PubsubMessage> input) |
@Nullable java.lang.String |
getIdAttribute()
Get the id attribute.
|
@Nullable java.lang.String |
getTimestampAttribute()
Get the timestamp attribute.
|
PubsubClient.TopicPath |
getTopic()
Get the topic being written to.
|
org.apache.beam.sdk.options.ValueProvider<PubsubClient.TopicPath> |
getTopicProvider()
Get the
ValueProvider for the topic being written to. |
public PubsubUnboundedSink(PubsubClient.PubsubClientFactory pubsubFactory, org.apache.beam.sdk.options.ValueProvider<PubsubClient.TopicPath> topic, java.lang.String timestampAttribute, java.lang.String idAttribute, int numShards)
public PubsubUnboundedSink(PubsubClient.PubsubClientFactory pubsubFactory, org.apache.beam.sdk.options.ValueProvider<PubsubClient.TopicPath> topic, java.lang.String timestampAttribute, java.lang.String idAttribute, int numShards, int publishBatchSize, int publishBatchBytes)
public PubsubClient.TopicPath getTopic()
public org.apache.beam.sdk.options.ValueProvider<PubsubClient.TopicPath> getTopicProvider()
ValueProvider
for the topic being written to.public @Nullable java.lang.String getTimestampAttribute()
public @Nullable java.lang.String getIdAttribute()
public org.apache.beam.sdk.values.PDone expand(org.apache.beam.sdk.values.PCollection<PubsubMessage> input)
expand
in class org.apache.beam.sdk.transforms.PTransform<org.apache.beam.sdk.values.PCollection<PubsubMessage>,org.apache.beam.sdk.values.PDone>