SparkUtils

Value Members

1. final def !=(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

2. final def ##(): Int

   

   Definition Classes

   AnyRef → Any

3. final def ==(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

4. final def asInstanceOf[T0]: T0

   

   Definition Classes

   Any

5. def clone(): AnyRef

   

   Attributes

   protected[java.lang]

   Definition Classes

   AnyRef

   Annotations

   @throws( ... )

6. def convertDataFrameToPrettyJSON(df: DataFrame, takeN: Int = 0): String

   

7. def convertDataframeFieldsToStrings(df: DataFrame): DataFrame

   

   Given an instance of DataFrame returns a dataframe where all primitive fields are converted to String

   Given an instance of DataFrame returns a dataframe where all primitive fields are converted to String

   df

   A dataframe

   returns

   A new dataframe with all primitive fields as Strings.

8. def currentActiveExecutors(sc: SparkContext): Seq[String]

   

   Retrieves all executors available for the current job.

9. final def eq(arg0: AnyRef): Boolean

   

   Definition Classes

   AnyRef

10. def equals(arg0: Any): Boolean

    

    Definition Classes

    AnyRef → Any

11. def finalize(): Unit

    

    Attributes

    protected[java.lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( classOf[java.lang.Throwable] )

12. def flattenSchema(df: DataFrame, useShortFieldNames: Boolean = false): DataFrame

    

    Given an instance of DataFrame returns a dataframe with flattened schema.

    Given an instance of DataFrame returns a dataframe with flattened schema. All nested structures are flattened and arrays are projected as columns.

    Note. The method checks the maximum size for each array and that could perform slowly, especially on a vary big dataframes.

    df

    A dataframe

    useShortFieldNames

    When flattening a schema each field name will contain full path. You can override this behavior and use a short field names instead

    returns

    A new dataframe with flat schema.

13. final def getClass(): Class[_]

    

    Definition Classes

    AnyRef → Any

14. def getDefaultHdfsBlockSize(spark: SparkSession): Option[Int]

    

15. def getField(path: String, schema: StructType): Option[StructField]

    

    Get a Spark field from a text path and a given schema (originally implemented here: https://github.com/AbsaOSS/enceladus/blob/665b34fa1c04fe255729e4b6706cf9ea33227b3e/utils/src/main/scala/za/co/absa/enceladus/utils/schema/SchemaUtils.scala#L45)

    Get a Spark field from a text path and a given schema (originally implemented here: https://github.com/AbsaOSS/enceladus/blob/665b34fa1c04fe255729e4b6706cf9ea33227b3e/utils/src/main/scala/za/co/absa/enceladus/utils/schema/SchemaUtils.scala#L45)

    path

    The dot-separated path to the field

    schema

    The schema which should contain the specified path

    returns

    Some(the requested field) or None if the field does not exist

16. def hashCode(): Int

    

    Definition Classes

    AnyRef → Any

17. final def isInstanceOf[T0]: Boolean

    

    Definition Classes

    Any

18. def logDebug(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

19. def logName: String

    

    Attributes

    protected

    Definition Classes

    Logging

20. def logger: Logger

    

    Attributes

    protected

    Definition Classes

    Logging

21. final def ne(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

22. final def notify(): Unit

    

    Definition Classes

    AnyRef

23. final def notifyAll(): Unit

    

    Definition Classes

    AnyRef

24. def prettyJSON(jsonIn: String): String

    

25. final def synchronized[T0](arg0: ⇒ T0): T0

    

    Definition Classes

    AnyRef

26. def toString(): String

    

    Definition Classes

    AnyRef → Any

27. final def wait(): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

28. final def wait(arg0: Long, arg1: Int): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

29. final def wait(arg0: Long): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )