FileSubFeedAction

abstract def breakFileRefLineage: Boolean

Stop propagating input FileRefs through action and instead get new FileRefs from DataObject according to the SubFeed's partitionValue.

abstract def deleteDataAfterRead(): Boolean

If true delete files after they are successfully processed.

abstract def doTransform(inputSubFeed: FileSubFeed, outputSubFeed: FileSubFeed, doExec: Boolean)(implicit session: SparkSession, context: ActionPipelineContext): FileSubFeed

"Transforms" a given FileSubFeed Note usage of doExec to choose between initialization or actual execution.

abstract def executionCondition: Option[Condition]

execution condition for this action.

abstract def executionMode: Option[ExecutionMode]

execution mode for this action.

abstract def factory: FromConfigFactory[Action]

Returns the factory that can parse this type (that is, type CO).

abstract val id: ActionId

A unique identifier for this instance.

abstract def input: FileRefDataObject with CanCreateInputStream

Input FileRefDataObject which can CanCreateInputStream

abstract def inputs: Seq[DataObject]

Input DataObjects To be implemented by subclasses

abstract def metadata: Option[ActionMetadata]

Additional metadata for the Action

abstract def metricsFailCondition: Option[String]

Spark SQL condition evaluated as where-clause against dataframe of metrics.

abstract def output: FileRefDataObject with CanCreateOutputStream

Output FileRefDataObject which can CanCreateOutputStream

abstract def outputs: Seq[DataObject]

Output DataObjects To be implemented by subclasses

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def addRuntimeEvent(executionId: ExecutionId, phase: ExecutionPhase, state: RuntimeEventState, msg: Option[String] = None, results: Seq[SubFeed] = Seq(), tstmp: LocalDateTime = LocalDateTime.now): Unit

Adds a runtime event for this Action

def addRuntimeMetrics(executionId: Option[ExecutionId], dataObjectId: Option[DataObjectId], metric: ActionMetrics): Unit

Adds a runtime metric for this Action

def applyExecutionMode(mainInput: DataObject, mainOutput: DataObject, subFeed: SubFeed, partitionValuesTransform: (Seq[PartitionValues]) ⇒ Map[PartitionValues, PartitionValues])(implicit session: SparkSession, context: ActionPipelineContext): Unit

Applies the executionMode and stores result in executionModeResult variable

final def asInstanceOf[T0]: T0

def atlasName: String

def atlasQualifiedName(prefix: String): String

def clone(): AnyRef

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

final def exec(subFeeds: Seq[SubFeed])(implicit session: SparkSession, context: ActionPipelineContext): Seq[SubFeed]

Action.exec implementation

var executionConditionResult: Option[(Boolean, Option[String])]

var executionModeResult: Option[Try[Option[ExecutionModeResult]]]

def finalize(): Unit

final def getClass(): Class[_]

def getDataObjectsState: Seq[DataObjectState]

Get potential state of input DataObjects when executionMode is DataObjectStateIncrementalMode.

def getInputDataObject[T <: DataObject](id: DataObjectId)(implicit arg0: ClassTag[T], arg1: scala.reflect.api.JavaUniverse.TypeTag[T], registry: InstanceRegistry): T

def getLatestRuntimeEventState: Option[RuntimeEventState]

Get latest runtime state

def getOutputDataObject[T <: DataObject](id: DataObjectId)(implicit arg0: ClassTag[T], arg1: scala.reflect.api.JavaUniverse.TypeTag[T], registry: InstanceRegistry): T

def getRuntimeDataImpl: RuntimeData

def getRuntimeInfo(executionId: Option[ExecutionId] = None): Option[RuntimeInfo]

Get summarized runtime information for a given ExecutionId.

def getRuntimeMetrics(executionId: Option[ExecutionId] = None): Map[DataObjectId, Option[ActionMetrics]]

Get the latest metrics for all DataObjects and a given SDLExecutionId.

def hashCode(): Int

final def init(subFeeds: Seq[SubFeed])(implicit session: SparkSession, context: ActionPipelineContext): Seq[SubFeed]

Action.init implementation

final def isInstanceOf[T0]: Boolean

lazy val logger: Logger

final def ne(arg0: AnyRef): Boolean

def nodeId: String

provide an implementation of the DAG node id

final def notify(): Unit

final def notifyAll(): Unit

final def postExec(inputSubFeeds: Seq[SubFeed], outputSubFeeds: Seq[SubFeed])(implicit session: SparkSession, context: ActionPipelineContext): Unit

Executes operations needed after executing an action.

def postExecFailed(implicit session: SparkSession): Unit

Executes operations needed to cleanup after executing an action failed.

def postExecSubFeed(inputSubFeed: SubFeed, outputSubFeed: SubFeed)(implicit session: SparkSession, context: ActionPipelineContext): Unit

def preExec(subFeeds: Seq[SubFeed])(implicit session: SparkSession, context: ActionPipelineContext): Unit

Executes operations needed before executing an action.

def preInit(subFeeds: Seq[SubFeed], dataObjectsState: Seq[DataObjectState])(implicit session: SparkSession, context: ActionPipelineContext): Unit

Checks before initalization of Action In this step execution condition is evaluated and Action init is skipped if result is false.

def prepare(implicit session: SparkSession, context: ActionPipelineContext): Unit

Prepare DataObjects prerequisites.

def recursiveInputs: Seq[FileRefDataObject with CanCreateInputStream]

Recursive Inputs on FileSubFeeds are not supported so empty Seq is set.

def setSparkJobMetadata(operation: Option[String] = None)(implicit session: SparkSession, context: ActionPipelineContext): Unit

Sets the util job description for better traceability in the Spark UI

final def synchronized[T0](arg0: ⇒ T0): T0

final def toString(executionId: Option[ExecutionId]): String

final def toString(): String

This is displayed in ascii graph visualization

def toStringMedium: String

def toStringShort: String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit