Packages

o

io.prophecy.libs

SparkTestingUtils

object SparkTestingUtils

Linear Supertypes
AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. SparkTestingUtils
  2. AnyRef
  3. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. All

Type Members

  1. class StringColumnExtensions extends AnyRef

Value Members

  1. final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2. final def ##(): Int
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  4. implicit def addDTMethods(s: String): StringColumnExtensions
  5. final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  6. def assertDFEquals(expectedUnsorted: DataFrame, resultUnsorted: DataFrame, maxUnequalRowsToShow: Int, tol: Double = 0.0): Either[Error, String]

    Compares if two DataFrames are equal, checks that the schemas are the same.

    Compares if two DataFrames are equal, checks that the schemas are the same. When comparing inexact fields uses tol.

    maxUnequalRowsToShow

    maximum unequal rows to expose

    tol

    max acceptable tolerance, should be less than 1.

  7. def clone(): AnyRef
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native() @HotSpotIntrinsicCandidate()
  8. def convertStringValueToDatatype(dataType: DataType, value: String): Any
  9. def createDF(spark: SparkSession, columns: Seq[String], values: Seq[Seq[Any]], defaults: Map[String, Any], typeMap: Map[String, DataType], port: String): DataFrame
  10. def createDfFromResourceFiles(spark: SparkSession, schemaDefinitionPath: String, dataPath: String, port: String): DataFrame
  11. def createDfFromSchemaFile(spark: SparkSession, schemaDefinitionPath: String, port: String, columns: Seq[String], values: Seq[Seq[Any]]): DataFrame
  12. def defaultForDatatype: PartialFunction[DataType, Any]
  13. def defaultsForSchema(schema: StructType): Map[String, Any]
  14. final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  15. def equals(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  16. final def getClass(): Class[_]
    Definition Classes
    AnyRef → Any
    Annotations
    @native() @HotSpotIntrinsicCandidate()
  17. def hashCode(): Int
    Definition Classes
    AnyRef → Any
    Annotations
    @native() @HotSpotIntrinsicCandidate()
  18. final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  19. def mkString(row: Row, sep: String): String
  20. final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  21. final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native() @HotSpotIntrinsicCandidate()
  22. final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native() @HotSpotIntrinsicCandidate()
  23. def nullify(df: DataFrame): DataFrame
  24. def postProcess(origDf: DataFrame): DataFrame
  25. def readData(spark: SparkSession, schema: StructType, path: String): DataFrame
  26. def readResource(path: String): String
  27. def rowEquals(r1: Row, r2: Row, tol: Double): Boolean
  28. def schemaToFieldMap(schema: StructType): Map[String, DataType]
  29. final def synchronized[T0](arg0: ⇒ T0): T0
    Definition Classes
    AnyRef
  30. def toString(): String
    Definition Classes
    AnyRef → Any
  31. final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  32. final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native()
  33. final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  34. def writeData(pathPrefix: String, fileName: String, df: DataFrame, separator: String, maxRows: Int = 100): Unit

Deprecated Value Members

  1. def finalize(): Unit
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] ) @Deprecated @deprecated
    Deprecated

    (Since version ) see corresponding Javadoc for more information.

Inherited from AnyRef

Inherited from Any

Ungrouped