SparkUtils

Value Members

1. final def !=(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

2. final def ##(): Int

   

   Definition Classes

   AnyRef → Any

3. final def ==(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

4. final def asInstanceOf[T0]: T0

   

   Definition Classes

   Any

5. def clone(): AnyRef

   

   Attributes

   protected[java.lang]

   Definition Classes

   AnyRef

   Annotations

   @throws( ... )

6. def convertDataFrameToPrettyJSON(df: DataFrame, takeN: Int = 0): String

   

7. def convertDataframeFieldsToStrings(df: DataFrame): DataFrame

   

   Given an instance of DataFrame returns a dataframe where all primitive fields are converted to String

   Given an instance of DataFrame returns a dataframe where all primitive fields are converted to String

   df

   A dataframe

   returns

   A new dataframe with all primitive fields as Strings.

8. def currentActiveExecutors(sc: SparkContext): Seq[String]

   

   Retrieves all executors available for the current job.

9. final def eq(arg0: AnyRef): Boolean

   

   Definition Classes

   AnyRef

10. def equals(arg0: Any): Boolean

    

    Definition Classes

    AnyRef → Any

11. def finalize(): Unit

    

    Attributes

    protected[java.lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( classOf[java.lang.Throwable] )

12. def flattenSchema(df: DataFrame, useShortFieldNames: Boolean = false): DataFrame

    

    Given an instance of DataFrame returns a dataframe with flattened schema.

    Given an instance of DataFrame returns a dataframe with flattened schema. All nested structures are flattened and arrays are projected as columns.

    Note. The method checks the maximum size for each array and that could perform slowly, especially on a vary big dataframes.

    df

    A dataframe

    useShortFieldNames

    When flattening a schema each field name will contain full path. You can override this behavior and use a short field names instead

    returns

    A new dataframe with flat schema.

13. final def getClass(): Class[_]

    

    Definition Classes

    AnyRef → Any

14. def hashCode(): Int

    

    Definition Classes

    AnyRef → Any

15. final def isInstanceOf[T0]: Boolean

    

    Definition Classes

    Any

16. final def ne(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

17. final def notify(): Unit

    

    Definition Classes

    AnyRef

18. final def notifyAll(): Unit

    

    Definition Classes

    AnyRef

19. def prettyJSON(jsonIn: String): String

    

20. final def synchronized[T0](arg0: ⇒ T0): T0

    

    Definition Classes

    AnyRef

21. def toString(): String

    

    Definition Classes

    AnyRef → Any

22. final def wait(): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

23. final def wait(arg0: Long, arg1: Int): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

24. final def wait(arg0: Long): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )