io

Type Members

1. case class BucketsConfiguration(number: Int, bucketColumns: Seq[String], sortByColumns: Seq[String] = Seq()) extends Product with Serializable

   

   Bucketing configuration

   Bucketing configuration

   number

   number of buckets

   bucketColumns

   columns for bucketing

   sortByColumns

   optional sort columns for bucketing

2. trait DataAwareSink[Config <: DataSinkConfiguration, WriteOut] extends AnyRef

   

   Common trait for writing an already defined data DataFrame to an external resource

3. trait DataAwareSinkFactory extends AnyRef

   

   Factory trait for DataAwareSinkFactory

4. trait DataSink[Config <: DataSinkConfiguration, WriteOut] extends AnyRef

   

   Common trait for writing a DataFrame to an external resource

5. trait DataSinkConfiguration extends AnyRef

   

   Common marker trait for DataSink configuration

6. case class DataSinkException(message: String = "", cause: Throwable = None.orNull) extends Exception with Product with Serializable

   

7. trait DataSource[Config <: DataSourceConfiguration] extends AnyRef

   

   Common trait for reading a DataFrame from an external resource

8. trait DataSourceConfiguration extends AnyRef

   

   Common marker trait for DataSource configuration

9. case class DataSourceException(message: String = "", cause: Throwable = None.orNull) extends Exception with Product with Serializable

   

10. trait DataSourceFactory extends AnyRef

    

    Factory trait for DataSourceFactory

11. case class FileDataAwareSink(configuration: FileSinkConfiguration, data: DataFrame) extends DataAwareSink[FileSinkConfiguration, DataFrame] with Product with Serializable

    

    FileDataSink trait that is data aware, so it can perform a write call with no arguments

12. case class FileDataSink(configuration: FileSinkConfiguration) extends DataSink[FileSinkConfiguration, DataFrame] with Logging with Product with Serializable

    

    FileDataSink trait

13. case class FileDataSource(configuration: FileSourceConfiguration) extends DataSource[FileSourceConfiguration] with Logging with Product with Serializable

    

14. case class FileSinkConfiguration(path: String, format: FormatType, optionalSaveMode: Option[String] = None, partitionFilesNumber: Option[Int] = None, partitionColumns: Seq[String] = Seq(), buckets: Option[BucketsConfiguration] = None, options: Map[String, String] = Map()) extends FormatAwareDataSinkConfiguration with Product with Serializable

    

    Output DataFrame sink configuration for Hadoop files.

    Output DataFrame sink configuration for Hadoop files.

    path

    the path of the target file

    format

    the format can be csv, json, orc, parquet, com.databricks.spark.avro or just avro and com.databricks.spark.xml or just xml

    optionalSaveMode

    the save mode can be overwrite, append, ignore and error; more details available at https://spark.apache.org/docs/2.3.1/api/java/org/apache/spark/sql/FileDataSink.html#mode-java.lang.String-

    partitionFilesNumber

    the number of partitions that the data will be partitioned to; if not given the number of partitions will be left unchanged

    partitionColumns

    optionally the writer can layout data in partitions similar to the hive partitions

    buckets

    optionally the writer can bucket the data, similar to Hive bucketing

    options

    other sink specific options

15. case class FileSourceConfiguration(path: String, sourceConfiguration: SourceConfiguration) extends FormatAwareDataSourceConfiguration with Product with Serializable

    

    Basic configuration for the FileDataSource

16. trait FormatAware extends AnyRef

    

    For things that should be aware of their format type

17. trait FormatAwareDataSinkConfiguration extends DataSinkConfiguration with FormatAware

    

    Common marker trait for DataSink configuration that also knows the data format

18. trait FormatAwareDataSourceConfiguration extends DataSourceConfiguration with FormatAware

    

    Common marker trait for DataSource configuration that also knows the data format

19. sealed trait FormatType extends AnyRef

    

20. case class JdbcDataAwareSink(configuration: JdbcSinkConfiguration, data: DataFrame) extends DataAwareSink[JdbcSinkConfiguration, DataFrame] with Product with Serializable

    

    JdbcDataSink trait that is data aware, so it can perform a write call with no arguments

21. case class JdbcDataSink(configuration: JdbcSinkConfiguration) extends DataSink[JdbcSinkConfiguration, DataFrame] with Logging with Product with Serializable

    

    JdbcDataSink trait

22. case class JdbcDataSource(configuration: JdbcSourceConfiguration) extends DataSource[JdbcSourceConfiguration] with Logging with Product with Serializable

    

23. case class JdbcSinkConfiguration(url: String, table: String, user: Option[String], password: Option[String], driver: Option[String], optionalSaveMode: Option[String], options: Map[String, String]) extends FormatAwareDataSinkConfiguration with Product with Serializable

    

    Basic configuration for the JdbcDataSource