Package org.apache.flink.streaming.api.graph

Class StreamingJobGraphGenerator

java.lang.Object

org.apache.flink.streaming.api.graph.StreamingJobGraphGenerator

@Internal
public class StreamingJobGraphGenerator
extends Object

The StreamingJobGraphGenerator converts a StreamGraph into a JobGraph.

Method Summary

Modifier and Type	Method	Description
static void	addVertexIndexPrefixInVertexName(JobVertexBuildContext jobVertexBuildContext, AtomicInteger vertexIndexId)
static IntermediateDataSet	connect(Integer headOfChain, StreamEdge edge, NonChainedOutput output, Map<Integer,JobVertex> jobVertices, JobVertexBuildContext jobVertexBuildContext)
static JobGraph	createAndInitializeJobGraph(StreamGraph streamGraph, org.apache.flink.api.common.JobID jobId)
static List<StreamEdge>	createChain(Integer currentNodeId, int chainIndex, OperatorChainInfo chainInfo, Map<Integer,OperatorChainInfo> chainEntryPoints, boolean canCreateNewChain, Executor serializationExecutor, JobVertexBuildContext jobVertexBuildContext, Consumer<Integer> visitedStreamNodeConsumer)
static org.apache.flink.api.common.operators.ResourceSpec	createChainedMinResources(Integer vertexID, List<StreamEdge> chainedOutputs, OperatorChainInfo operatorChainInfo, JobVertexBuildContext jobVertexBuildContext)
static String	createChainedName(Integer vertexID, List<StreamEdge> chainedOutputs, Optional<OperatorChainInfo> operatorChainInfo, Map<Integer,String> chainedNames, JobVertexBuildContext jobVertexBuildContext)
static org.apache.flink.api.common.operators.ResourceSpec	createChainedPreferredResources(Integer vertexID, List<StreamEdge> chainedOutputs, OperatorChainInfo operatorChainInfo, JobVertexBuildContext jobVertexBuildContext)
static JobGraph	createJobGraph(ClassLoader userClassLoader, StreamGraph streamGraph, org.apache.flink.api.common.JobID jobID)
static JobGraph	createJobGraph(StreamGraph streamGraph)
static JobVertex	createJobVertex(OperatorChainInfo chainInfo, Executor serializationExecutor, JobVertexBuildContext jobVertexBuildContext)
static void	createSourceChainInfo(StreamNode sourceNode, Map<Integer,OperatorChainInfo> chainEntryPoints, JobVertexBuildContext jobVertexBuildContext)
static ResultPartitionType	determineUndefinedResultPartitionType(StreamEdge edge, JobVertexBuildContext jobVertexBuildContext)
static OperatorID	generateOperatorID(String operatorUid)	Creates an instance of OperatorID based on the provided operator unique identifier (UID).
static boolean	isChainable(StreamEdge edge, StreamGraph streamGraph)
static boolean	isChainable(StreamEdge edge, StreamGraph streamGraph, boolean allowChainWithDefaultParallelism)
static boolean	isChainableSource(StreamNode streamNode, StreamGraph streamGraph)
static void	markSupportingConcurrentExecutionAttempts(JobVertexBuildContext jobVertexBuildContext)
static void	preValidate(StreamGraph streamGraph, ClassLoader userClassloader)
static void	serializeOperatorCoordinatorsAndStreamConfig(Executor serializationExecutor, JobVertexBuildContext jobVertexBuildContext)
static void	setAllOperatorNonChainedOutputsConfigs(Map<Integer,Map<StreamEdge,NonChainedOutput>> opIntermediateOutputs, JobVertexBuildContext jobVertexBuildContext)
static void	setManagedMemoryFraction(JobVertexBuildContext jobVertexBuildContext)
static void	setOperatorChainedOutputsConfig(StreamConfig config, List<StreamEdge> chainableOutputs, JobVertexBuildContext jobVertexBuildContext)
static void	setOperatorConfig(Integer vertexId, StreamConfig config, OperatorChainInfo chainInfo, JobVertexBuildContext jobVertexBuildContext)
static void	setPhysicalEdges(JobVertexBuildContext jobVertexBuildContext)
static void	setSlotSharingAndCoLocation(JobVertexBuildContext jobVertexBuildContext)
static void	setVertexDescription(JobVertexBuildContext jobVertexBuildContext)
static void	tryConvertPartitionerForDynamicGraph(List<StreamEdge> chainableOutputs, List<StreamEdge> nonChainableOutputs, JobVertexBuildContext jobVertexBuildContext)
static void	validateHybridShuffleExecuteInBatchMode(JobVertexBuildContext jobVertexBuildContext)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Method Detail

createJobGraph

@VisibleForTesting
public static JobGraph createJobGraph(StreamGraph streamGraph)

createJobGraph

public static JobGraph createJobGraph(ClassLoader userClassLoader,
                                      StreamGraph streamGraph,
                                      @Nullable
                                      org.apache.flink.api.common.JobID jobID)

serializeOperatorCoordinatorsAndStreamConfig

public static void serializeOperatorCoordinatorsAndStreamConfig(Executor serializationExecutor,
                                                                JobVertexBuildContext jobVertexBuildContext)

generateOperatorID

public static OperatorID generateOperatorID(String operatorUid)

Creates an instance of OperatorID based on the provided operator unique identifier (UID).

Parameters:

operatorUid - the unique identifier of the operator, used to generate the hash

Returns:

a new OperatorID instance generated from the specified operator UID

addVertexIndexPrefixInVertexName

public static void addVertexIndexPrefixInVertexName(JobVertexBuildContext jobVertexBuildContext,
                                                    AtomicInteger vertexIndexId)

setVertexDescription

public static void setVertexDescription(JobVertexBuildContext jobVertexBuildContext)

preValidate

public static void preValidate(StreamGraph streamGraph,
                               ClassLoader userClassloader)

setPhysicalEdges

public static void setPhysicalEdges(JobVertexBuildContext jobVertexBuildContext)

createChain

public static List<StreamEdge> createChain(Integer currentNodeId,
                                           int chainIndex,
                                           OperatorChainInfo chainInfo,
                                           Map<Integer,OperatorChainInfo> chainEntryPoints,
                                           boolean canCreateNewChain,
                                           Executor serializationExecutor,
                                           JobVertexBuildContext jobVertexBuildContext,
                                           @Nullable
                                           Consumer<Integer> visitedStreamNodeConsumer)

createAndInitializeJobGraph

public static JobGraph createAndInitializeJobGraph(StreamGraph streamGraph,
                                                   @Nullable
                                                   org.apache.flink.api.common.JobID jobId)

createSourceChainInfo

public static void createSourceChainInfo(StreamNode sourceNode,
                                         Map<Integer,OperatorChainInfo> chainEntryPoints,
                                         JobVertexBuildContext jobVertexBuildContext)

createChainedName

public static String createChainedName(Integer vertexID,
                                       List<StreamEdge> chainedOutputs,
                                       Optional<OperatorChainInfo> operatorChainInfo,
                                       Map<Integer,String> chainedNames,
                                       JobVertexBuildContext jobVertexBuildContext)

createChainedMinResources

public static org.apache.flink.api.common.operators.ResourceSpec createChainedMinResources(Integer vertexID,
                                                                                           List<StreamEdge> chainedOutputs,
                                                                                           OperatorChainInfo operatorChainInfo,
                                                                                           JobVertexBuildContext jobVertexBuildContext)

createChainedPreferredResources

public static org.apache.flink.api.common.operators.ResourceSpec createChainedPreferredResources(Integer vertexID,
                                                                                                 List<StreamEdge> chainedOutputs,
                                                                                                 OperatorChainInfo operatorChainInfo,
                                                                                                 JobVertexBuildContext jobVertexBuildContext)

createJobVertex

public static JobVertex createJobVertex(OperatorChainInfo chainInfo,
                                        Executor serializationExecutor,
                                        JobVertexBuildContext jobVertexBuildContext)

setOperatorConfig

public static void setOperatorConfig(Integer vertexId,
                                     StreamConfig config,
                                     OperatorChainInfo chainInfo,
                                     JobVertexBuildContext jobVertexBuildContext)

setOperatorChainedOutputsConfig

public static void setOperatorChainedOutputsConfig(StreamConfig config,
                                                   List<StreamEdge> chainableOutputs,
                                                   JobVertexBuildContext jobVertexBuildContext)

setAllOperatorNonChainedOutputsConfigs

public static void setAllOperatorNonChainedOutputsConfigs(Map<Integer,Map<StreamEdge,NonChainedOutput>> opIntermediateOutputs,
                                                          JobVertexBuildContext jobVertexBuildContext)

tryConvertPartitionerForDynamicGraph

public static void tryConvertPartitionerForDynamicGraph(List<StreamEdge> chainableOutputs,
                                                        List<StreamEdge> nonChainableOutputs,
                                                        JobVertexBuildContext jobVertexBuildContext)

connect

public static IntermediateDataSet connect(Integer headOfChain,
                                          StreamEdge edge,
                                          NonChainedOutput output,
                                          Map<Integer,JobVertex> jobVertices,
                                          JobVertexBuildContext jobVertexBuildContext)

determineUndefinedResultPartitionType

public static ResultPartitionType determineUndefinedResultPartitionType(StreamEdge edge,
                                                                        JobVertexBuildContext jobVertexBuildContext)

isChainable

public static boolean isChainable(StreamEdge edge,
                                  StreamGraph streamGraph)

isChainable

public static boolean isChainable(StreamEdge edge,
                                  StreamGraph streamGraph,
                                  boolean allowChainWithDefaultParallelism)

isChainableSource

public static boolean isChainableSource(StreamNode streamNode,
                                        StreamGraph streamGraph)

markSupportingConcurrentExecutionAttempts

public static void markSupportingConcurrentExecutionAttempts(JobVertexBuildContext jobVertexBuildContext)

setSlotSharingAndCoLocation

public static void setSlotSharingAndCoLocation(JobVertexBuildContext jobVertexBuildContext)

validateHybridShuffleExecuteInBatchMode

public static void validateHybridShuffleExecuteInBatchMode(JobVertexBuildContext jobVertexBuildContext)

setManagedMemoryFraction

public static void setManagedMemoryFraction(JobVertexBuildContext jobVertexBuildContext)