IngestionJob

Abstract Value Members

abstract def domain: Domain
abstract def ingest(dataset: DataFrame): (RDD[_], RDD[_])

ingestion algorithm
ingestion algorithm

Attributes
protected
abstract def loadDataSet(): Try[DataFrame]

Dataset loading strategy (JSON / CSV / ...)
Dataset loading strategy (JSON / CSV / ...)
returns
Spark Dataframe loaded using metadata options

Attributes
protected
abstract def name: String

Definition Classes
JobBase
abstract def options: Map[String, String]
abstract def path: List[Path]
abstract def schema: Schema
abstract def schemaHandler: SchemaHandler
implicit abstract def settings: Settings

Definition Classes
JobBase
abstract def storageHandler: StorageHandler
abstract def types: List[Type]

Concrete Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def analyze(fullTableName: String): Any

Attributes
protected
Definition Classes
SparkJob
def appendToFile(storageHandler: StorageHandler, dataToSave: DataFrame, path: Path): Unit

Saves a dataset.
Saves a dataset. If the path is empty (the first time we call metrics on the schema) then we can write.
If there's already parquet files stored in it, then create a temporary directory to compute on, and flush the path to move updated metrics in it
dataToSave
: dataset to be saved
path
: Path to save the file at

Attributes
protected
Definition Classes
SparkJob
def applyIgnore(dfIn: DataFrame): Dataset[Row]

Attributes
protected
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def createViews(views: Views, sqlParameters: Map[String, String], activeEnv: Map[String, String]): Unit

Attributes
protected
Definition Classes
SparkJob
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getWriteMode(): WriteMode
def hashCode(): Int

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
val logger: Logger

Attributes
protected
Definition Classes
StrictLogging
lazy val metadata: Metadata

Merged metadata
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
val now: Timestamp
def partitionDataset(dataset: DataFrame, partition: List[String]): DataFrame

Attributes
protected
Definition Classes
SparkJob
def partitionedDatasetWriter(dataset: DataFrame, partition: List[String]): DataFrameWriter[Row]

Partition a dataset using dataset columns.
Partition a dataset using dataset columns. To partition the dataset using the ingestion time, use the reserved column names :
- comet_date
- comet_year
- comet_month
- comet_day
- comet_hour
- comet_minute These columns are renamed to "date", "year", "month", "day", "hour", "minute" in the dataset and their values is set to the current date/time.
dataset
: Input dataset
partition
: list of columns to use for partitioning.
returns
The Spark session used to run this job

Attributes
protected
Definition Classes
SparkJob
def registerUdf(udf: String): Unit

Attributes
protected
Definition Classes
SparkJob
def run(): Try[JobResult]

Main entry point as required by the Spark Job interface
Main entry point as required by the Spark Job interface
returns
: Spark Session used for the job

Definition Classes
IngestionJob → JobBase
def saveAccepted(acceptedDF: DataFrame): (DataFrame, Path)

Merge new and existing dataset if required Save using overwrite / Append mode
Merge new and existing dataset if required Save using overwrite / Append mode

Attributes
protected
def saveRejected(rejectedRDD: RDD[String]): Try[Path]

Attributes
protected
lazy val session: SparkSession

Definition Classes
SparkJob
lazy val sparkEnv: SparkEnv

Definition Classes
SparkJob
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Doc: package ingest

trait IngestionJob extends SparkJob

Abstract Value Members

abstract def domain: Domain

abstract def ingest(dataset: DataFrame): (RDD[_], RDD[_])

abstract def loadDataSet(): Try[DataFrame]

abstract def name: String

abstract def options: Map[String, String]

abstract def path: List[Path]

abstract def schema: Schema

abstract def schemaHandler: SchemaHandler

implicit abstract def settings: Settings

abstract def storageHandler: StorageHandler

abstract def types: List[Type]

Concrete Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def analyze(fullTableName: String): Any

def appendToFile(storageHandler: StorageHandler, dataToSave: DataFrame, path: Path): Unit

def applyIgnore(dfIn: DataFrame): Dataset[Row]

final def asInstanceOf[T0]: T0

def clone(): AnyRef

def createViews(views: Views, sqlParameters: Map[String, String], activeEnv: Map[String, String]): Unit

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def getClass(): Class[_]

def getWriteMode(): WriteMode

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

val logger: Logger

lazy val metadata: Metadata

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

val now: Timestamp

def partitionDataset(dataset: DataFrame, partition: List[String]): DataFrame

def partitionedDatasetWriter(dataset: DataFrame, partition: List[String]): DataFrameWriter[Row]

def registerUdf(udf: String): Unit

def run(): Try[JobResult]

def saveAccepted(acceptedDF: DataFrame): (DataFrame, Path)

def saveRejected(rejectedRDD: RDD[String]): Try[Path]

lazy val session: SparkSession

lazy val sparkEnv: SparkEnv

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from SparkJob

Inherited from JobBase

Inherited from StrictLogging

Inherited from AnyRef

Inherited from Any

Ungrouped