org.apache.beam.sdk.util.construction

Class ParDoTranslation

java.lang.Object

org.apache.beam.sdk.util.construction.ParDoTranslation

public class ParDoTranslation
extends java.lang.Object

Utilities for interacting with ParDo instances and RunnerApi.ParDoPayload protos.

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static interface	ParDoTranslation.ParDoLike These methods drive to-proto translation from Java and from rehydrated ParDos.
static class	ParDoTranslation.ParDoLikeTimerFamilySpecs
static class	ParDoTranslation.ParDoTranslator A PTransformTranslation.TransformPayloadTranslator for ParDo.

Field Summary

Fields

Modifier and Type	Field and Description
static java.lang.String	BAG_USER_STATE Represents a user state specification that supports a bag.
static java.lang.String	CUSTOM_JAVA_DO_FN_URN The URN for an unknown Java DoFn.
static java.lang.String	CUSTOM_JAVA_VIEW_FN_URN The URN for an unknown Java ViewFn.
static java.lang.String	CUSTOM_JAVA_WINDOW_MAPPING_FN_URN The URN for an unknown Java WindowMappingFn.
static java.lang.String	MULTIMAP_USER_STATE Represents a user state specification that supports a multimap.
static java.lang.String	REQUIRES_BUNDLE_FINALIZATION_URN This requirement indicates the requests_finalization field of ParDo transform payloads must be inspected.
static java.lang.String	REQUIRES_ON_WINDOW_EXPIRATION_URN This requirement indicates that the ParDo requires a callback on each window expiration.
static java.lang.String	REQUIRES_SPLITTABLE_DOFN_URN This requirement indicates the restriction_coder_id field of ParDo transform payloads must be inspected.
static java.lang.String	REQUIRES_STABLE_INPUT_URN This requirement indicates the requires_stable_input field of ParDo transform payloads must be inspected.
static java.lang.String	REQUIRES_STATEFUL_PROCESSING_URN This requirement indicates the state_spec and time_spec fields of ParDo transform payloads must be inspected.
static java.lang.String	REQUIRES_TIME_SORTED_INPUT_URN This requirement indicates the requires_time_sorted_input field of ParDo transform payloads must be inspected.

Constructor Summary

Constructors

Constructor and Description
ParDoTranslation()

Method Summary

Modifier and Type	Method and Description
static DoFnWithExecutionInformation	doFnWithExecutionInformationFromProto(org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec fnSpec)
static TupleTagList	getAdditionalOutputTags(AppliedPTransform<?,?,?> application)
static DoFn<?,?>	getDoFn(AppliedPTransform<?,?,?> application)
static DoFn<?,?>	getDoFn(org.apache.beam.model.pipeline.v1.RunnerApi.ParDoPayload payload)
static org.apache.beam.model.pipeline.v1.RunnerApi.PCollection	getMainInput(org.apache.beam.model.pipeline.v1.RunnerApi.PTransform ptransform, org.apache.beam.model.pipeline.v1.RunnerApi.Components components)
static java.lang.String	getMainInputName(org.apache.beam.model.pipeline.v1.RunnerApi.PTransformOrBuilder ptransform) Returns the name of the main input of the ptransform.
static TupleTag<?>	getMainOutputTag(AppliedPTransform<?,?,?> application)
static TupleTag<?>	getMainOutputTag(org.apache.beam.model.pipeline.v1.RunnerApi.ParDoPayload payload)
static java.util.Map<TupleTag<?>,Coder<?>>	getOutputCoders(AppliedPTransform<?,?,?> application)
static DoFnSchemaInformation	getSchemaInformation(AppliedPTransform<?,?,?> application)
static DoFnSchemaInformation	getSchemaInformation(org.apache.beam.model.pipeline.v1.RunnerApi.ParDoPayload payload)
static DoFnSchemaInformation	getSchemaInformation(org.apache.beam.model.pipeline.v1.RunnerApi.PTransform pTransform)
static java.util.Map<java.lang.String,PCollectionView<?>>	getSideInputMapping(AppliedPTransform<?,?,?> application)
static java.util.Map<java.lang.String,PCollectionView<?>>	getSideInputMapping(org.apache.beam.model.pipeline.v1.RunnerApi.ParDoPayload payload)
static java.util.Map<java.lang.String,PCollectionView<?>>	getSideInputMapping(org.apache.beam.model.pipeline.v1.RunnerApi.PTransform pTransform)
static java.util.List<PCollectionView<?>>	getSideInputs(AppliedPTransform<?,?,?> application)
static boolean	isSplittable(AppliedPTransform<?,?,?> transform)
static org.apache.beam.model.pipeline.v1.RunnerApi.ParDoPayload	payloadForParDoLike(ParDoTranslation.ParDoLike parDo, SdkComponents components)
static java.lang.String	registerCoderOrThrow(SdkComponents components, Coder coder)
static org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec	translateDoFn(DoFn<?,?> fn, TupleTag<?> tag, java.util.Map<java.lang.String,PCollectionView<?>> sideInputMapping, DoFnSchemaInformation doFnSchemaInformation, SdkComponents components)
static org.apache.beam.model.pipeline.v1.RunnerApi.ParDoPayload	translateParDo(AppliedPTransform<?,?,ParDo.MultiOutput<?,?>> appliedPTransform, SdkComponents components)
static <InputT> org.apache.beam.model.pipeline.v1.RunnerApi.ParDoPayload	translateParDo(ParDo.MultiOutput<InputT,?> parDo, PCollection<InputT> mainInput, DoFnSchemaInformation doFnSchemaInformation, Pipeline pipeline, SdkComponents components) Translate a ParDo.
static java.util.Map<java.lang.String,org.apache.beam.model.pipeline.v1.RunnerApi.SideInput>	translateSideInputs(java.util.List<PCollectionView<?>> views, SdkComponents components)
static org.apache.beam.model.pipeline.v1.RunnerApi.StateSpec	translateStateSpec(StateSpec<?> stateSpec, SdkComponents components) Translate state specs.
static org.apache.beam.model.pipeline.v1.RunnerApi.TimeDomain.Enum	translateTimeDomain(TimeDomain timeDomain)
static org.apache.beam.model.pipeline.v1.RunnerApi.TimerFamilySpec	translateTimerFamilySpec(TimerSpec timer, SdkComponents components, Coder<?> keyCoder, Coder<BoundedWindow> windowCoder)
static org.apache.beam.model.pipeline.v1.RunnerApi.SideInput	translateView(PCollectionView<?> view, SdkComponents components)
static org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec	translateViewFn(ViewFn<?,?> viewFn, SdkComponents components)
static org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec	translateWindowMappingFn(WindowMappingFn<?> windowMappingFn, SdkComponents components)
static boolean	usesStateOrTimers(AppliedPTransform<?,?,?> transform)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

REQUIRES_STATEFUL_PROCESSING_URN

public static final java.lang.String REQUIRES_STATEFUL_PROCESSING_URN

This requirement indicates the state_spec and time_spec fields of ParDo transform payloads must be inspected.

See Also:

Constant Field Values

REQUIRES_BUNDLE_FINALIZATION_URN

public static final java.lang.String REQUIRES_BUNDLE_FINALIZATION_URN

This requirement indicates the requests_finalization field of ParDo transform payloads must be inspected.

See Also:

Constant Field Values

REQUIRES_STABLE_INPUT_URN

public static final java.lang.String REQUIRES_STABLE_INPUT_URN

This requirement indicates the requires_stable_input field of ParDo transform payloads must be inspected.

See Also:

Constant Field Values

REQUIRES_TIME_SORTED_INPUT_URN

public static final java.lang.String REQUIRES_TIME_SORTED_INPUT_URN

This requirement indicates the requires_time_sorted_input field of ParDo transform payloads must be inspected.

See Also:

Constant Field Values

REQUIRES_SPLITTABLE_DOFN_URN

public static final java.lang.String REQUIRES_SPLITTABLE_DOFN_URN

This requirement indicates the restriction_coder_id field of ParDo transform payloads must be inspected.

See Also:

Constant Field Values

REQUIRES_ON_WINDOW_EXPIRATION_URN

public static final java.lang.String REQUIRES_ON_WINDOW_EXPIRATION_URN

This requirement indicates that the ParDo requires a callback on each window expiration.

See Also:

Constant Field Values

BAG_USER_STATE

public static final java.lang.String BAG_USER_STATE

Represents a user state specification that supports a bag.

See Also:

Constant Field Values

MULTIMAP_USER_STATE

public static final java.lang.String MULTIMAP_USER_STATE

Represents a user state specification that supports a multimap.

See Also:

Constant Field Values

CUSTOM_JAVA_DO_FN_URN

public static final java.lang.String CUSTOM_JAVA_DO_FN_URN

The URN for an unknown Java DoFn.

See Also:

Constant Field Values

CUSTOM_JAVA_VIEW_FN_URN

public static final java.lang.String CUSTOM_JAVA_VIEW_FN_URN

The URN for an unknown Java ViewFn.

See Also:

Constant Field Values

CUSTOM_JAVA_WINDOW_MAPPING_FN_URN

public static final java.lang.String CUSTOM_JAVA_WINDOW_MAPPING_FN_URN

The URN for an unknown Java WindowMappingFn.

See Also:

Constant Field Values

Constructor Detail

ParDoTranslation

public ParDoTranslation()

Method Detail

translateParDo

public static org.apache.beam.model.pipeline.v1.RunnerApi.ParDoPayload translateParDo(AppliedPTransform<?,?,ParDo.MultiOutput<?,?>> appliedPTransform,
                                                                                      SdkComponents components)
                                                                               throws java.io.IOException

Throws:

java.io.IOException

translateParDo

public static <InputT> org.apache.beam.model.pipeline.v1.RunnerApi.ParDoPayload translateParDo(ParDo.MultiOutput<InputT,?> parDo,
                                                                                               PCollection<InputT> mainInput,
                                                                                               DoFnSchemaInformation doFnSchemaInformation,
                                                                                               Pipeline pipeline,
                                                                                               SdkComponents components)
                                                                                        throws java.io.IOException

Translate a ParDo.

Throws:

java.io.IOException

getDoFn

public static DoFn<?,?> getDoFn(org.apache.beam.model.pipeline.v1.RunnerApi.ParDoPayload payload)
                         throws org.apache.beam.vendor.grpc.v1p60p1.com.google.protobuf.InvalidProtocolBufferException

Throws:

org.apache.beam.vendor.grpc.v1p60p1.com.google.protobuf.InvalidProtocolBufferException

getDoFn

public static DoFn<?,?> getDoFn(AppliedPTransform<?,?,?> application)
                         throws java.io.IOException

Throws:

java.io.IOException

getSchemaInformation

public static DoFnSchemaInformation getSchemaInformation(AppliedPTransform<?,?,?> application)

getSchemaInformation

public static DoFnSchemaInformation getSchemaInformation(org.apache.beam.model.pipeline.v1.RunnerApi.PTransform pTransform)

getSchemaInformation

public static DoFnSchemaInformation getSchemaInformation(org.apache.beam.model.pipeline.v1.RunnerApi.ParDoPayload payload)

getMainOutputTag

public static TupleTag<?> getMainOutputTag(org.apache.beam.model.pipeline.v1.RunnerApi.ParDoPayload payload)
                                    throws org.apache.beam.vendor.grpc.v1p60p1.com.google.protobuf.InvalidProtocolBufferException

Throws:

org.apache.beam.vendor.grpc.v1p60p1.com.google.protobuf.InvalidProtocolBufferException

getSideInputMapping

public static java.util.Map<java.lang.String,PCollectionView<?>> getSideInputMapping(AppliedPTransform<?,?,?> application)

getSideInputMapping

public static java.util.Map<java.lang.String,PCollectionView<?>> getSideInputMapping(org.apache.beam.model.pipeline.v1.RunnerApi.PTransform pTransform)

getSideInputMapping

public static java.util.Map<java.lang.String,PCollectionView<?>> getSideInputMapping(org.apache.beam.model.pipeline.v1.RunnerApi.ParDoPayload payload)

getMainOutputTag

public static TupleTag<?> getMainOutputTag(AppliedPTransform<?,?,?> application)
                                    throws java.io.IOException

Throws:

java.io.IOException

getAdditionalOutputTags

public static TupleTagList getAdditionalOutputTags(AppliedPTransform<?,?,?> application)
                                            throws java.io.IOException

Throws:

java.io.IOException

getOutputCoders

public static java.util.Map<TupleTag<?>,Coder<?>> getOutputCoders(AppliedPTransform<?,?,?> application)

getSideInputs

public static java.util.List<PCollectionView<?>> getSideInputs(AppliedPTransform<?,?,?> application)
                                                        throws java.io.IOException

Throws:

java.io.IOException

getMainInput

public static org.apache.beam.model.pipeline.v1.RunnerApi.PCollection getMainInput(org.apache.beam.model.pipeline.v1.RunnerApi.PTransform ptransform,
                                                                                   org.apache.beam.model.pipeline.v1.RunnerApi.Components components)
                                                                            throws java.io.IOException

Throws:

java.io.IOException

getMainInputName

public static java.lang.String getMainInputName(org.apache.beam.model.pipeline.v1.RunnerApi.PTransformOrBuilder ptransform)
                                         throws java.io.IOException

Returns the name of the main input of the ptransform.

Throws:

java.io.IOException

translateStateSpec

public static org.apache.beam.model.pipeline.v1.RunnerApi.StateSpec translateStateSpec(StateSpec<?> stateSpec,
                                                                                       SdkComponents components)
                                                                                throws java.io.IOException

Translate state specs.

Throws:

java.io.IOException

registerCoderOrThrow

public static java.lang.String registerCoderOrThrow(SdkComponents components,
                                                    Coder coder)

translateTimerFamilySpec

public static org.apache.beam.model.pipeline.v1.RunnerApi.TimerFamilySpec translateTimerFamilySpec(TimerSpec timer,
                                                                                                   SdkComponents components,
                                                                                                   Coder<?> keyCoder,
                                                                                                   Coder<BoundedWindow> windowCoder)

translateTimeDomain

public static org.apache.beam.model.pipeline.v1.RunnerApi.TimeDomain.Enum translateTimeDomain(TimeDomain timeDomain)

translateDoFn

public static org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec translateDoFn(DoFn<?,?> fn,
                                                                                     TupleTag<?> tag,
                                                                                     java.util.Map<java.lang.String,PCollectionView<?>> sideInputMapping,
                                                                                     DoFnSchemaInformation doFnSchemaInformation,
                                                                                     SdkComponents components)

doFnWithExecutionInformationFromProto

public static DoFnWithExecutionInformation doFnWithExecutionInformationFromProto(org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec fnSpec)

translateSideInputs

public static java.util.Map<java.lang.String,org.apache.beam.model.pipeline.v1.RunnerApi.SideInput> translateSideInputs(java.util.List<PCollectionView<?>> views,
                                                                                                                        SdkComponents components)

translateView

public static org.apache.beam.model.pipeline.v1.RunnerApi.SideInput translateView(PCollectionView<?> view,
                                                                                  SdkComponents components)

translateViewFn

public static org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec translateViewFn(ViewFn<?,?> viewFn,
                                                                                       SdkComponents components)

usesStateOrTimers

public static boolean usesStateOrTimers(AppliedPTransform<?,?,?> transform)
                                 throws java.io.IOException

Throws:

java.io.IOException

isSplittable

public static boolean isSplittable(AppliedPTransform<?,?,?> transform)
                            throws java.io.IOException

Throws:

java.io.IOException

translateWindowMappingFn

public static org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec translateWindowMappingFn(WindowMappingFn<?> windowMappingFn,
                                                                                                SdkComponents components)

payloadForParDoLike

public static org.apache.beam.model.pipeline.v1.RunnerApi.ParDoPayload payloadForParDoLike(ParDoTranslation.ParDoLike parDo,
                                                                                           SdkComponents components)
                                                                                    throws java.io.IOException

Throws:

java.io.IOException