AccessTableDataObject

Instance Constructors

new AccessTableDataObject(id: DataObjectId, path: String, schemaMin: Option[StructType] = None, table: Table, metadata: Option[DataObjectMetadata] = None)(implicit instanceRegistry: InstanceRegistry)

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def addFieldIfNotExisting(writeSchema: StructType, colName: String, dataType: DataType): StructType

Attributes
protected
Definition Classes
CanCreateDataFrame
final def asInstanceOf[T0]: T0

Definition Classes
Any
def atlasName: String

Definition Classes
TableDataObject → DataObject → AtlasExportable
def atlasQualifiedName(prefix: String): String

Definition Classes
TableDataObject → AtlasExportable
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def createReadSchema(writeSchema: StructType)(implicit session: SparkSession): StructType

Creates the read schema based on a given write schema.
Creates the read schema based on a given write schema. Normally this is the same, but some DataObjects can remove & add columns on read (e.g. KafkaTopicDataObject, SparkFileDataObject) In this cases we have to break the DataFrame lineage und create a dummy DataFrame in init phase.

Definition Classes
CanCreateDataFrame
def dropTable(implicit session: SparkSession): Unit

Definition Classes
AccessTableDataObject → TableDataObject
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def factory: FromConfigFactory[DataObject]

Returns the factory that can parse this type (that is, type CO).
Returns the factory that can parse this type (that is, type CO).
Typically, implementations of this method should return the companion object of the implementing class. The companion object in turn should implement FromConfigFactory.
returns
the factory (object) for this class.

Definition Classes
AccessTableDataObject → ParsableFromConfig
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getConnection[T <: Connection](connectionId: ConnectionId)(implicit registry: InstanceRegistry, ct: ClassTag[T], tt: scala.reflect.api.JavaUniverse.TypeTag[T]): T

Handle class cast exception when getting objects from instance registry
Handle class cast exception when getting objects from instance registry

Attributes
protected
Definition Classes
DataObject
def getConnectionReg[T <: Connection](connectionId: ConnectionId, registry: InstanceRegistry)(implicit ct: ClassTag[T], tt: scala.reflect.api.JavaUniverse.TypeTag[T]): T

Attributes
protected
Definition Classes
DataObject
def getDataFrame(partitionValues: Seq[PartitionValues] = Seq())(implicit session: SparkSession, context: ActionPipelineContext): DataFrame

Definition Classes
AccessTableDataObject → CanCreateDataFrame
def getDataFrameByFramework(partitionValues: Seq[PartitionValues] = Seq())(implicit session: SparkSession, context: ActionPipelineContext): DataFrame
def getPKduplicates(implicit session: SparkSession, context: ActionPipelineContext): DataFrame

Definition Classes
TableDataObject
def getPKnulls(implicit session: SparkSession, context: ActionPipelineContext): DataFrame

Definition Classes
TableDataObject
def getPKviolators(implicit session: SparkSession, context: ActionPipelineContext): DataFrame

Definition Classes
TableDataObject
def housekeepingMode: Option[HousekeepingMode]

Configure a housekeeping mode to e.g cleanup, archive and compact partitions.
Configure a housekeeping mode to e.g cleanup, archive and compact partitions. Default is None.

Definition Classes
DataObject
val id: DataObjectId

A unique identifier for this instance.
A unique identifier for this instance.

Definition Classes
AccessTableDataObject → DataObject → SdlConfigObject
implicit val instanceRegistry: InstanceRegistry
def isDbExisting(implicit session: SparkSession): Boolean

Definition Classes
AccessTableDataObject → TableDataObject
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def isPKcandidateKey(implicit session: SparkSession, context: ActionPipelineContext): Boolean

Definition Classes
TableDataObject
def isTableExisting(implicit session: SparkSession): Boolean

Definition Classes
AccessTableDataObject → TableDataObject
lazy val logger: Logger

Attributes
protected
Definition Classes
SmartDataLakeLogger
val metadata: Option[DataObjectMetadata]

Additional metadata for the DataObject
Additional metadata for the DataObject

Definition Classes
AccessTableDataObject → DataObject
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
val path: String
val schemaMin: Option[StructType]

An optional, minimal schema that a DataObject schema must have to pass schema validation.
An optional, minimal schema that a DataObject schema must have to pass schema validation.
The schema validation semantics are: - Schema A is valid in respect to a minimal schema B when B is a subset of A. This means: the whole column set of B is contained in the column set of A.
- A column of B is contained in A when A contains a column with equal name and data type.
- Column order is ignored.
- Column nullability is ignored.
- Duplicate columns in terms of name and data type are eliminated (set semantics).
Note: This is mainly used by the functionality defined in CanCreateDataFrame and CanWriteDataFrame, that is, when reading or writing Spark data frames from/to the underlying data container. io.smartdatalake.workflow.action.Actions that work with files ignore the schemaMin attribute if it is defined. Additionally schemaMin can be used to define the schema used if there is no data or table doesn't yet exist.
Definition Classes
AccessTableDataObject → SchemaValidation
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
var table: Table

Definition Classes
AccessTableDataObject → TableDataObject
var tableSchema: StructType

Definition Classes
TableDataObject
def toStringShort: String

Definition Classes
DataObject
def validateSchema(df: DataFrame, schemaExpected: StructType, role: String): Unit

Validate the schema of a given Spark Data Frame df against a given expected schema.
Validate the schema of a given Spark Data Frame df against a given expected schema.
df
The data frame to validate.
schemaExpected
The expected schema to validate against.
role
role used in exception message. Set to read or write.

Definition Classes
SchemaValidation
Exceptions thrown
SchemaViolationException is the schemaMin does not validate.
def validateSchemaMin(df: DataFrame, role: String): Unit

Validate the schema of a given Spark Data Frame df against schemaMin.
Validate the schema of a given Spark Data Frame df against schemaMin.
df
The data frame to validate.
role
role used in exception message. Set to read or write.

Definition Classes
SchemaValidation
Exceptions thrown
SchemaViolationException is the schemaMin does not validate.
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Docs: object AccessTableDataObject | package dataobject

case class AccessTableDataObject(id: DataObjectId, path: String, schemaMin: Option[StructType] = None, table: Table, metadata: Option[DataObjectMetadata] = None)(implicit instanceRegistry: InstanceRegistry) extends TableDataObject with Product with Serializable

Instance Constructors

new AccessTableDataObject(id: DataObjectId, path: String, schemaMin: Option[StructType] = None, table: Table, metadata: Option[DataObjectMetadata] = None)(implicit instanceRegistry: InstanceRegistry)

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def addFieldIfNotExisting(writeSchema: StructType, colName: String, dataType: DataType): StructType

final def asInstanceOf[T0]: T0

def atlasName: String

def atlasQualifiedName(prefix: String): String

def clone(): AnyRef

def createReadSchema(writeSchema: StructType)(implicit session: SparkSession): StructType

def dropTable(implicit session: SparkSession): Unit

final def eq(arg0: AnyRef): Boolean

def factory: FromConfigFactory[DataObject]

def finalize(): Unit

final def getClass(): Class[_]

def getConnection[T <: Connection](connectionId: ConnectionId)(implicit registry: InstanceRegistry, ct: ClassTag[T], tt: scala.reflect.api.JavaUniverse.TypeTag[T]): T

def getConnectionReg[T <: Connection](connectionId: ConnectionId, registry: InstanceRegistry)(implicit ct: ClassTag[T], tt: scala.reflect.api.JavaUniverse.TypeTag[T]): T

def getDataFrame(partitionValues: Seq[PartitionValues] = Seq())(implicit session: SparkSession, context: ActionPipelineContext): DataFrame

def getDataFrameByFramework(partitionValues: Seq[PartitionValues] = Seq())(implicit session: SparkSession, context: ActionPipelineContext): DataFrame

def getPKduplicates(implicit session: SparkSession, context: ActionPipelineContext): DataFrame

def getPKnulls(implicit session: SparkSession, context: ActionPipelineContext): DataFrame

def getPKviolators(implicit session: SparkSession, context: ActionPipelineContext): DataFrame

def housekeepingMode: Option[HousekeepingMode]

val id: DataObjectId

implicit val instanceRegistry: InstanceRegistry

def isDbExisting(implicit session: SparkSession): Boolean

final def isInstanceOf[T0]: Boolean

def isPKcandidateKey(implicit session: SparkSession, context: ActionPipelineContext): Boolean

def isTableExisting(implicit session: SparkSession): Boolean

lazy val logger: Logger

val metadata: Option[DataObjectMetadata]

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

val path: String

val schemaMin: Option[StructType]

final def synchronized[T0](arg0: ⇒ T0): T0

var table: Table

var tableSchema: StructType

def toStringShort: String

def validateSchema(df: DataFrame, schemaExpected: StructType, role: String): Unit

def validateSchemaMin(df: DataFrame, role: String): Unit

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from Serializable

Inherited from Serializable

Inherited from Product

Inherited from Equals

Inherited from TableDataObject

Inherited from SchemaValidation

Inherited from CanCreateDataFrame

Inherited from DataObject

Inherited from AtlasExportable

Inherited from SmartDataLakeLogger

Inherited from ParsableFromConfig[DataObject]

Inherited from SdlConfigObject

Inherited from AnyRef

Inherited from Any

Ungrouped