Class

org.apache.spark.sql.hive

HiveContext

Related Doc: package hive

Permalink

class HiveContext extends SQLContext with Logging

An instance of the Spark SQL execution engine that integrates with data stored in Hive. Configuration for Hive is read from hive-site.xml on the classpath.

Self Type
HiveContext
Annotations
@deprecated
Deprecated

(Since version 2.0.0) Use SparkSession.builder.enableHiveSupport instead

Linear Supertypes
SQLContext, Serializable, Serializable, Logging, AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. HiveContext
  2. SQLContext
  3. Serializable
  4. Serializable
  5. Logging
  6. AnyRef
  7. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. All

Instance Constructors

  1. new HiveContext(sc: JavaSparkContext)

    Permalink
  2. new HiveContext(sc: SparkContext)

    Permalink

Value Members

  1. final def !=(arg0: Any): Boolean

    Permalink
    Definition Classes
    AnyRef → Any
  2. final def ##(): Int

    Permalink
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean

    Permalink
    Definition Classes
    AnyRef → Any
  4. final def asInstanceOf[T0]: T0

    Permalink
    Definition Classes
    Any
  5. def baseRelationToDataFrame(baseRelation: BaseRelation): DataFrame

    Permalink
    Definition Classes
    SQLContext
  6. def cacheTable(tableName: String): Unit

    Permalink
    Definition Classes
    SQLContext
  7. def clearCache(): Unit

    Permalink
    Definition Classes
    SQLContext
  8. def clone(): AnyRef

    Permalink
    Attributes
    protected[java.lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  9. def createDataFrame(data: List[_], beanClass: Class[_]): DataFrame

    Permalink
    Definition Classes
    SQLContext
  10. def createDataFrame(rdd: JavaRDD[_], beanClass: Class[_]): DataFrame

    Permalink
    Definition Classes
    SQLContext
  11. def createDataFrame(rdd: RDD[_], beanClass: Class[_]): DataFrame

    Permalink
    Definition Classes
    SQLContext
  12. def createDataFrame(rows: List[Row], schema: StructType): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @DeveloperApi()
  13. def createDataFrame(rowRDD: JavaRDD[Row], schema: StructType): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @DeveloperApi()
  14. def createDataFrame(rowRDD: RDD[Row], schema: StructType): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @DeveloperApi()
  15. def createDataFrame[A <: Product](data: Seq[A])(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @Experimental()
  16. def createDataFrame[A <: Product](rdd: RDD[A])(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @Experimental()
  17. def createDataset[T](data: List[T])(implicit arg0: Encoder[T]): Dataset[T]

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @Experimental()
  18. def createDataset[T](data: RDD[T])(implicit arg0: Encoder[T]): Dataset[T]

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @Experimental()
  19. def createDataset[T](data: Seq[T])(implicit arg0: Encoder[T]): Dataset[T]

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @Experimental()
  20. def createExternalTable(tableName: String, source: String, schema: StructType, options: Map[String, String]): DataFrame

    Permalink
    Definition Classes
    SQLContext
  21. def createExternalTable(tableName: String, source: String, schema: StructType, options: Map[String, String]): DataFrame

    Permalink
    Definition Classes
    SQLContext
  22. def createExternalTable(tableName: String, source: String, options: Map[String, String]): DataFrame

    Permalink
    Definition Classes
    SQLContext
  23. def createExternalTable(tableName: String, source: String, options: Map[String, String]): DataFrame

    Permalink
    Definition Classes
    SQLContext
  24. def createExternalTable(tableName: String, path: String, source: String): DataFrame

    Permalink
    Definition Classes
    SQLContext
  25. def createExternalTable(tableName: String, path: String): DataFrame

    Permalink
    Definition Classes
    SQLContext
  26. def dropTempTable(tableName: String): Unit

    Permalink
    Definition Classes
    SQLContext
  27. def emptyDataFrame: DataFrame

    Permalink
    Definition Classes
    SQLContext
  28. final def eq(arg0: AnyRef): Boolean

    Permalink
    Definition Classes
    AnyRef
  29. def equals(arg0: Any): Boolean

    Permalink
    Definition Classes
    AnyRef → Any
  30. def experimental: ExperimentalMethods

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @Experimental() @transient()
  31. def finalize(): Unit

    Permalink
    Attributes
    protected[java.lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] )
  32. def getAllConfs: Map[String, String]

    Permalink
    Definition Classes
    SQLContext
  33. final def getClass(): Class[_]

    Permalink
    Definition Classes
    AnyRef → Any
  34. def getConf(key: String, defaultValue: String): String

    Permalink
    Definition Classes
    SQLContext
  35. def getConf(key: String): String

    Permalink
    Definition Classes
    SQLContext
  36. def hashCode(): Int

    Permalink
    Definition Classes
    AnyRef → Any
  37. def initializeLogIfNecessary(isInterpreter: Boolean): Unit

    Permalink
    Attributes
    protected
    Definition Classes
    Logging
  38. def isCached(tableName: String): Boolean

    Permalink
    Definition Classes
    SQLContext
  39. final def isDebugEnabled: Boolean

    Permalink
    Attributes
    protected
    Definition Classes
    Logging
  40. final def isInfoEnabled: Boolean

    Permalink
    Attributes
    protected
    Definition Classes
    Logging
  41. final def isInstanceOf[T0]: Boolean

    Permalink
    Definition Classes
    Any
  42. final def isTraceEnabled: Boolean

    Permalink
    Attributes
    protected
    Definition Classes
    Logging
  43. def listenerManager: ExecutionListenerManager

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @Experimental()
  44. def log: Logger

    Permalink
    Attributes
    protected
    Definition Classes
    Logging
  45. def logDebug(msg: ⇒ String, throwable: Throwable): Unit

    Permalink
    Attributes
    protected
    Definition Classes
    Logging
  46. def logDebug(msg: ⇒ String): Unit

    Permalink
    Attributes
    protected
    Definition Classes
    Logging
  47. def logError(msg: ⇒ String, throwable: Throwable): Unit

    Permalink
    Attributes
    protected
    Definition Classes
    Logging
  48. def logError(msg: ⇒ String): Unit

    Permalink
    Attributes
    protected
    Definition Classes
    Logging
  49. def logInfo(msg: ⇒ String, throwable: Throwable): Unit

    Permalink
    Attributes
    protected
    Definition Classes
    Logging
  50. def logInfo(msg: ⇒ String): Unit

    Permalink
    Attributes
    protected
    Definition Classes
    Logging
  51. def logName: String

    Permalink
    Attributes
    protected
    Definition Classes
    Logging
  52. def logTrace(msg: ⇒ String, throwable: Throwable): Unit

    Permalink
    Attributes
    protected
    Definition Classes
    Logging
  53. def logTrace(msg: ⇒ String): Unit

    Permalink
    Attributes
    protected
    Definition Classes
    Logging
  54. def logWarning(msg: ⇒ String, throwable: Throwable): Unit

    Permalink
    Attributes
    protected
    Definition Classes
    Logging
  55. def logWarning(msg: ⇒ String): Unit

    Permalink
    Attributes
    protected
    Definition Classes
    Logging
  56. final def ne(arg0: AnyRef): Boolean

    Permalink
    Definition Classes
    AnyRef
  57. def newSession(): HiveContext

    Permalink

    Returns a new HiveContext as new session, which will have separated SQLConf, UDF/UDAF, temporary tables and SessionState, but sharing the same CacheManager, IsolatedClientLoader and Hive client (both of execution and metadata) with existing HiveContext.

    Returns a new HiveContext as new session, which will have separated SQLConf, UDF/UDAF, temporary tables and SessionState, but sharing the same CacheManager, IsolatedClientLoader and Hive client (both of execution and metadata) with existing HiveContext.

    Definition Classes
    HiveContext → SQLContext
  58. final def notify(): Unit

    Permalink
    Definition Classes
    AnyRef
  59. final def notifyAll(): Unit

    Permalink
    Definition Classes
    AnyRef
  60. def range(start: Long, end: Long, step: Long, numPartitions: Int): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @Experimental()
  61. def range(start: Long, end: Long, step: Long): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @Experimental()
  62. def range(start: Long, end: Long): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @Experimental()
  63. def range(end: Long): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @Experimental()
  64. def read: DataFrameReader

    Permalink
    Definition Classes
    SQLContext
  65. def readStream: DataStreamReader

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @Experimental()
  66. def refreshTable(tableName: String): Unit

    Permalink

    Invalidate and refresh all the cached the metadata of the given table.

    Invalidate and refresh all the cached the metadata of the given table. For performance reasons, Spark SQL or the external data source library it uses might cache certain metadata about a table, such as the location of blocks. When those change outside of Spark SQL, users should call this function to invalidate the cache.

    Since

    1.3.0

  67. def sessionState: HiveSessionState

    Permalink
    Attributes
    protected[org.apache.spark.sql]
    Definition Classes
    HiveContext → SQLContext
  68. def setConf(key: String, value: String): Unit

    Permalink
    Definition Classes
    SQLContext
  69. def setConf(props: Properties): Unit

    Permalink
    Definition Classes
    SQLContext
  70. def sharedState: HiveSharedState

    Permalink
    Attributes
    protected[org.apache.spark.sql]
    Definition Classes
    HiveContext → SQLContext
  71. def sparkContext: SparkContext

    Permalink
    Definition Classes
    SQLContext
  72. val sparkSession: SparkSession

    Permalink
    Definition Classes
    SQLContext
  73. def sql(sqlText: String): DataFrame

    Permalink
    Definition Classes
    SQLContext
  74. def streams: StreamingQueryManager

    Permalink
    Definition Classes
    SQLContext
  75. final def synchronized[T0](arg0: ⇒ T0): T0

    Permalink
    Definition Classes
    AnyRef
  76. def table(tableName: String): DataFrame

    Permalink
    Definition Classes
    SQLContext
  77. def tableNames(databaseName: String): Array[String]

    Permalink
    Definition Classes
    SQLContext
  78. def tableNames(): Array[String]

    Permalink
    Definition Classes
    SQLContext
  79. def tables(databaseName: String): DataFrame

    Permalink
    Definition Classes
    SQLContext
  80. def tables(): DataFrame

    Permalink
    Definition Classes
    SQLContext
  81. def toString(): String

    Permalink
    Definition Classes
    AnyRef → Any
  82. def udf: UDFRegistration

    Permalink
    Definition Classes
    SQLContext
  83. def uncacheTable(tableName: String): Unit

    Permalink
    Definition Classes
    SQLContext
  84. final def wait(): Unit

    Permalink
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  85. final def wait(arg0: Long, arg1: Int): Unit

    Permalink
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  86. final def wait(arg0: Long): Unit

    Permalink
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

Deprecated Value Members

  1. def applySchema(rdd: JavaRDD[_], beanClass: Class[_]): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.3.0) Use createDataFrame instead.

  2. def applySchema(rdd: RDD[_], beanClass: Class[_]): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.3.0) Use createDataFrame instead.

  3. def applySchema(rowRDD: JavaRDD[Row], schema: StructType): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.3.0) Use createDataFrame instead.

  4. def applySchema(rowRDD: RDD[Row], schema: StructType): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.3.0) Use createDataFrame instead.

  5. def jdbc(url: String, table: String, theParts: Array[String]): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.jdbc() instead.

  6. def jdbc(url: String, table: String, columnName: String, lowerBound: Long, upperBound: Long, numPartitions: Int): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.jdbc() instead.

  7. def jdbc(url: String, table: String): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.jdbc() instead.

  8. def jsonFile(path: String, samplingRatio: Double): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.json() instead.

  9. def jsonFile(path: String, schema: StructType): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.json() instead.

  10. def jsonFile(path: String): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.json() instead.

  11. def jsonRDD(json: JavaRDD[String], samplingRatio: Double): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.json() instead.

  12. def jsonRDD(json: RDD[String], samplingRatio: Double): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.json() instead.

  13. def jsonRDD(json: JavaRDD[String], schema: StructType): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.json() instead.

  14. def jsonRDD(json: RDD[String], schema: StructType): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.json() instead.

  15. def jsonRDD(json: JavaRDD[String]): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.json() instead.

  16. def jsonRDD(json: RDD[String]): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.json() instead.

  17. def load(source: String, schema: StructType, options: Map[String, String]): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.format(source).schema(schema).options(options).load() instead.

  18. def load(source: String, schema: StructType, options: Map[String, String]): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.format(source).schema(schema).options(options).load() instead.

  19. def load(source: String, options: Map[String, String]): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.format(source).options(options).load() instead.

  20. def load(source: String, options: Map[String, String]): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.format(source).options(options).load() instead.

  21. def load(path: String, source: String): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.format(source).load(path) instead.

  22. def load(path: String): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @deprecated
    Deprecated

    (Since version 1.4.0) Use read.load(path) instead.

  23. def parquetFile(paths: String*): DataFrame

    Permalink
    Definition Classes
    SQLContext
    Annotations
    @deprecated @varargs()
    Deprecated

    (Since version 1.4.0) Use read.parquet() instead.

Inherited from SQLContext

Inherited from Serializable

Inherited from Serializable

Inherited from Logging

Inherited from AnyRef

Inherited from Any

Ungrouped