org.apache.spark.sql.hive.test

TestHiveContext

class TestHiveContext extends SQLContext

A locally running test instance of Spark's Hive execution engine.

Data from testTables will be automatically loaded whenever a query is run over those tables. Calling reset will delete all tables and other state in the database, leaving the database in a "clean" state.

TestHive is singleton object version of this class because instantiating multiple copies of the hive metastore seems to lead to weird non-deterministic failures. Therefore, the execution of test cases that rely on TestHive must be serialized.

Linear Supertypes
SQLContext, Serializable, Serializable, Logging, AnyRef, Any
Known Subclasses
TestHive
Ordering
  1. Alphabetic
  2. By inheritance
Inherited
  1. TestHiveContext
  2. SQLContext
  3. Serializable
  4. Serializable
  5. Logging
  6. AnyRef
  7. Any
  1. Hide All
  2. Show all
Learn more about member selection
Visibility
  1. Public
  2. All

Instance Constructors

  1. new TestHiveContext(sc: SparkContext, loadTestTables: Boolean = true)

    If loadTestTables is false, no test tables are loaded.

    If loadTestTables is false, no test tables are loaded. Note that this flag can only be true when running in the JVM, i.e. it needs to be false when calling from Python.

  2. new TestHiveContext(sparkSession: TestHiveSparkSession)

Value Members

  1. final def !=(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef

  2. final def !=(arg0: Any): Boolean

    Definition Classes
    Any

  3. final def ##(): Int

    Definition Classes
    AnyRef → Any

  4. final def ==(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef

  5. final def ==(arg0: Any): Boolean

    Definition Classes
    Any

  6. final def asInstanceOf[T0]: T0

    Definition Classes
    Any

  7. def baseRelationToDataFrame(baseRelation: BaseRelation): DataFrame

    Definition Classes
    SQLContext

  8. def cacheTable(tableName: String): Unit

    Definition Classes
    SQLContext

  9. def clearCache(): Unit

    Definition Classes
    SQLContext

  10. def clone(): AnyRef

    Attributes
    protected[java.lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

  11. def createDataFrame(data: List[_], beanClass: Class[_]): DataFrame

    Definition Classes
    SQLContext

  12. def createDataFrame(rdd: JavaRDD[_], beanClass: Class[_]): DataFrame

    Definition Classes
    SQLContext

  13. def createDataFrame(rdd: RDD[_], beanClass: Class[_]): DataFrame

    Definition Classes
    SQLContext

  14. def createDataFrame(rows: List[Row], schema: StructType): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @DeveloperApi() @Evolving()

  15. def createDataFrame(rowRDD: JavaRDD[Row], schema: StructType): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @DeveloperApi() @Evolving()

  16. def createDataFrame(rowRDD: RDD[Row], schema: StructType): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @DeveloperApi() @Evolving()

  17. def createDataFrame[A <: Product](data: Seq[A])(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @Experimental() @Evolving()

  18. def createDataFrame[A <: Product](rdd: RDD[A])(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @Experimental() @Evolving()

  19. def createDataset[T](data: List[T])(implicit arg0: Encoder[T]): Dataset[T]

    Definition Classes
    SQLContext
    Annotations
    @Experimental() @Evolving()

  20. def createDataset[T](data: RDD[T])(implicit arg0: Encoder[T]): Dataset[T]

    Definition Classes
    SQLContext
    Annotations
    @Experimental()

  21. def createDataset[T](data: Seq[T])(implicit arg0: Encoder[T]): Dataset[T]

    Definition Classes
    SQLContext
    Annotations
    @Experimental() @Evolving()

  22. def createExternalTable(tableName: String, source: String, schema: StructType, options: Map[String, String]): DataFrame

    Definition Classes
    SQLContext

  23. def createExternalTable(tableName: String, source: String, schema: StructType, options: Map[String, String]): DataFrame

    Definition Classes
    SQLContext

  24. def createExternalTable(tableName: String, source: String, options: Map[String, String]): DataFrame

    Definition Classes
    SQLContext

  25. def createExternalTable(tableName: String, source: String, options: Map[String, String]): DataFrame

    Definition Classes
    SQLContext

  26. def createExternalTable(tableName: String, path: String, source: String): DataFrame

    Definition Classes
    SQLContext

  27. def createExternalTable(tableName: String, path: String): DataFrame

    Definition Classes
    SQLContext

  28. def dropTempTable(tableName: String): Unit

    Definition Classes
    SQLContext

  29. def emptyDataFrame: DataFrame

    Definition Classes
    SQLContext

  30. final def eq(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef

  31. def equals(arg0: Any): Boolean

    Definition Classes
    AnyRef → Any

  32. def experimental: ExperimentalMethods

    Definition Classes
    SQLContext
    Annotations
    @Experimental() @transient() @Unstable()

  33. def finalize(): Unit

    Attributes
    protected[java.lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] )

  34. def getAllConfs: Map[String, String]

    Definition Classes
    SQLContext

  35. final def getClass(): Class[_]

    Definition Classes
    AnyRef → Any

  36. def getConf(key: String, defaultValue: String): String

    Definition Classes
    SQLContext

  37. def getConf(key: String): String

    Definition Classes
    SQLContext

  38. def getHiveFile(path: String): File

  39. def hashCode(): Int

    Definition Classes
    AnyRef → Any

  40. def initializeLogIfNecessary(isInterpreter: Boolean): Unit

    Attributes
    protected
    Definition Classes
    Logging

  41. def isCached(tableName: String): Boolean

    Definition Classes
    SQLContext

  42. final def isInstanceOf[T0]: Boolean

    Definition Classes
    Any

  43. def isTraceEnabled(): Boolean

    Attributes
    protected
    Definition Classes
    Logging

  44. def listenerManager: ExecutionListenerManager

    Definition Classes
    SQLContext
    Annotations
    @Experimental() @Evolving()

  45. def loadTestTable(name: String): Unit

  46. def log: Logger

    Attributes
    protected
    Definition Classes
    Logging

  47. def logDebug(msg: ⇒ String, throwable: Throwable): Unit

    Attributes
    protected
    Definition Classes
    Logging

  48. def logDebug(msg: ⇒ String): Unit

    Attributes
    protected
    Definition Classes
    Logging

  49. def logError(msg: ⇒ String, throwable: Throwable): Unit

    Attributes
    protected
    Definition Classes
    Logging

  50. def logError(msg: ⇒ String): Unit

    Attributes
    protected
    Definition Classes
    Logging

  51. def logInfo(msg: ⇒ String, throwable: Throwable): Unit

    Attributes
    protected
    Definition Classes
    Logging

  52. def logInfo(msg: ⇒ String): Unit

    Attributes
    protected
    Definition Classes
    Logging

  53. def logName: String

    Attributes
    protected
    Definition Classes
    Logging

  54. def logTrace(msg: ⇒ String, throwable: Throwable): Unit

    Attributes
    protected
    Definition Classes
    Logging

  55. def logTrace(msg: ⇒ String): Unit

    Attributes
    protected
    Definition Classes
    Logging

  56. def logWarning(msg: ⇒ String, throwable: Throwable): Unit

    Attributes
    protected
    Definition Classes
    Logging

  57. def logWarning(msg: ⇒ String): Unit

    Attributes
    protected
    Definition Classes
    Logging

  58. final def ne(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef

  59. def newSession(): TestHiveContext

    Definition Classes
    TestHiveContext → SQLContext

  60. final def notify(): Unit

    Definition Classes
    AnyRef

  61. final def notifyAll(): Unit

    Definition Classes
    AnyRef

  62. def range(start: Long, end: Long, step: Long, numPartitions: Int): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @Experimental() @Evolving()

  63. def range(start: Long, end: Long, step: Long): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @Experimental() @Evolving()

  64. def range(start: Long, end: Long): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @Experimental() @Evolving()

  65. def range(end: Long): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @Experimental() @Evolving()

  66. def read: DataFrameReader

    Definition Classes
    SQLContext

  67. def readStream: DataStreamReader

    Definition Classes
    SQLContext
    Annotations
    @Experimental() @Evolving()

  68. def reset(): Unit

  69. def sessionState: TestHiveSessionState

    Definition Classes
    TestHiveContext → SQLContext

  70. def setCacheTables(c: Boolean): Unit

  71. def setConf(key: String, value: String): Unit

    Definition Classes
    SQLContext

  72. def setConf(props: Properties): Unit

    Definition Classes
    SQLContext

  73. def sparkContext: SparkContext

    Definition Classes
    SQLContext

  74. val sparkSession: TestHiveSparkSession

    Definition Classes
    TestHiveContext → SQLContext

  75. def sql(sqlText: String): DataFrame

    Definition Classes
    SQLContext

  76. def streams: StreamingQueryManager

    Definition Classes
    SQLContext

  77. final def synchronized[T0](arg0: ⇒ T0): T0

    Definition Classes
    AnyRef

  78. def table(tableName: String): DataFrame

    Definition Classes
    SQLContext

  79. def tableNames(databaseName: String): Array[String]

    Definition Classes
    SQLContext

  80. def tableNames(): Array[String]

    Definition Classes
    SQLContext

  81. def tables(databaseName: String): DataFrame

    Definition Classes
    SQLContext

  82. def tables(): DataFrame

    Definition Classes
    SQLContext

  83. def toString(): String

    Definition Classes
    AnyRef → Any

  84. def udf: UDFRegistration

    Definition Classes
    SQLContext

  85. def uncacheTable(tableName: String): Unit

    Definition Classes
    SQLContext

  86. final def wait(): Unit

    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

  87. final def wait(arg0: Long, arg1: Int): Unit

    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

  88. final def wait(arg0: Long): Unit

    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

Deprecated Value Members

  1. def applySchema(rdd: JavaRDD[_], beanClass: Class[_]): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.3.0) Use createDataFrame instead.

  2. def applySchema(rdd: RDD[_], beanClass: Class[_]): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.3.0) Use createDataFrame instead.

  3. def applySchema(rowRDD: JavaRDD[Row], schema: StructType): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.3.0) Use createDataFrame instead.

  4. def applySchema(rowRDD: RDD[Row], schema: StructType): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.3.0) Use createDataFrame instead.

  5. def jdbc(url: String, table: String, theParts: Array[String]): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.jdbc() instead.

  6. def jdbc(url: String, table: String, columnName: String, lowerBound: Long, upperBound: Long, numPartitions: Int): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.jdbc() instead.

  7. def jdbc(url: String, table: String): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.jdbc() instead.

  8. def jsonFile(path: String, samplingRatio: Double): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.json() instead.

  9. def jsonFile(path: String, schema: StructType): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.json() instead.

  10. def jsonFile(path: String): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.json() instead.

  11. def jsonRDD(json: JavaRDD[String], samplingRatio: Double): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.json() instead.

  12. def jsonRDD(json: RDD[String], samplingRatio: Double): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.json() instead.

  13. def jsonRDD(json: JavaRDD[String], schema: StructType): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.json() instead.

  14. def jsonRDD(json: RDD[String], schema: StructType): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.json() instead.

  15. def jsonRDD(json: JavaRDD[String]): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.json() instead.

  16. def jsonRDD(json: RDD[String]): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.json() instead.

  17. def load(source: String, schema: StructType, options: Map[String, String]): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.format(source).schema(schema).options(options).load() instead.

  18. def load(source: String, schema: StructType, options: Map[String, String]): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.format(source).schema(schema).options(options).load() instead.

  19. def load(source: String, options: Map[String, String]): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.format(source).options(options).load() instead.

  20. def load(source: String, options: Map[String, String]): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.format(source).options(options).load() instead.

  21. def load(path: String, source: String): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.format(source).load(path) instead.

  22. def load(path: String): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.load(path) instead.

  23. def parquetFile(paths: String*): DataFrame

    Definition Classes
    SQLContext
    Annotations
    @deprecated @varargs()
    Deprecated

    (Since version 1.4.0) Use read.parquet() instead.

Inherited from SQLContext

Inherited from Serializable

Inherited from Serializable

Inherited from Logging

Inherited from AnyRef

Inherited from Any

Ungrouped