HiveSource

Instance Constructors

new HiveSource(dbName: String, tableName: String, projection: List[String] = Nil, predicate: Option[Predicate] = None, partitionConstraint: Option[PartitionConstraint] = None, principal: Option[String] = None, keytabPath: Option[Path] = None)(implicit fs: FileSystem, client: IMetaStoreClient)

projection
sets which fields are required by the caller.
predicate
optional predicate which will filter rows at the read level

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
implicit val conf: Configuration
val dbName: String
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def files(): Map[Path, Seq[Path]]

Returns all the files used by this table.
Returns all the files used by this table. The result is a mapping of partition path to the files contained in that partition.
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def isPartitionOnlyProjection(): Boolean
val keytabPath: Option[Path]
val logger: Logger

Attributes
protected
Definition Classes
Logging
lazy val metastoreSchema: StructType
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
val ops: HiveOps
val partitionConstraint: Option[PartitionConstraint]
def parts(): List[Part]

Definition Classes
HiveSource → Source
def paths(includePartitionDirs: Boolean = false, includeTableDir: Boolean = false): List[Path]

Returns a list of all files used by this hive source.
Returns a list of all files used by this hive source.
includePartitionDirs
if true then the partition directories will be included
includeTableDir
if true then the main table directory will be included
returns
paths of all files and directories
val predicate: Option[Predicate]

optional predicate which will filter rows at the read level
val principal: Option[String]
val projection: List[String]

sets which fields are required by the caller.
def schema: StructType

The returned schema should take into account:
The returned schema should take into account:
1) Any projection. If a projection is set, then it should return the schema in the same order as the projection. If no projection is set then the schema should be driven from the hive metastore.
2) Any partitions set. These should be included in the schema columns.

Definition Classes
HiveSource → Source
def setAcl(acl: AclSpec, includePartitionDirs: Boolean = false, includeTableDir: Boolean = false): Unit

Sets the acl for all files of this hive source.
Sets the acl for all files of this hive source. Even if the files are not located inside the table directory, this function will find them and correctly update the spec.
acl
the acl values to set
def setPermissions(permission: FsPermission, includePartitionDirs: Boolean = false, includeTableDir: Boolean = false): Unit
def showDdl(ifNotExists: Boolean = true): String
def spec(): TableSpec

Returns a TableSpec which contains details of the underlying table.
Returns a TableSpec which contains details of the underlying table. Similar to the Table class in the Hive API but using scala friendly types.
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
val tableName: String
def tablePermission(): FsPermission
def toFrame(_listener: Listener): Frame

Definition Classes
Source
def toFrame(): Frame

Definition Classes
Source
def using[T, U <: AnyRef { def close(): Unit }](closeable: U)(f: (U) ⇒ T): T

Definition Classes
Using
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
def withKeytabFile(principal: String, keytabPath: Path): HiveSource
def withPartitionConstraint(constraint: PartitionConstraint): HiveSource
def withPredicate(predicate: Predicate): HiveSource
def withProjection(columns: Seq[String]): HiveSource
def withProjection(first: String, rest: String*): HiveSource

Deprecated Value Members

def select(first: String, rest: String*): HiveSource

Annotations
@deprecated
Deprecated
(Since version 1.1.0) use withProjection()

Related Doc: package hive

Instance Constructors

new HiveSource(dbName: String, tableName: String, projection: List[String] = Nil, predicate: Option[Predicate] = None, partitionConstraint: Option[PartitionConstraint] = None, principal: Option[String] = None, keytabPath: Option[Path] = None)(implicit fs: FileSystem, client: IMetaStoreClient)

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def clone(): AnyRef

implicit val conf: Configuration

val dbName: String

final def eq(arg0: AnyRef): Boolean

def files(): Map[Path, Seq[Path]]

def finalize(): Unit

final def getClass(): Class[_]

final def isInstanceOf[T0]: Boolean

def isPartitionOnlyProjection(): Boolean

val keytabPath: Option[Path]

val logger: Logger

lazy val metastoreSchema: StructType

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

val ops: HiveOps

val partitionConstraint: Option[PartitionConstraint]

def parts(): List[Part]

def paths(includePartitionDirs: Boolean = false, includeTableDir: Boolean = false): List[Path]

val predicate: Option[Predicate]

val principal: Option[String]

val projection: List[String]

def schema: StructType

def setAcl(acl: AclSpec, includePartitionDirs: Boolean = false, includeTableDir: Boolean = false): Unit

def setPermissions(permission: FsPermission, includePartitionDirs: Boolean = false, includeTableDir: Boolean = false): Unit

def showDdl(ifNotExists: Boolean = true): String

def spec(): TableSpec

final def synchronized[T0](arg0: ⇒ T0): T0

val tableName: String

def tablePermission(): FsPermission

def toFrame(_listener: Listener): Frame

def toFrame(): Frame

def using[T, U <: AnyRef { def close(): Unit }](closeable: U)(f: (U) ⇒ T): T

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

def withKeytabFile(principal: String, keytabPath: Path): HiveSource

def withPartitionConstraint(constraint: PartitionConstraint): HiveSource

def withPredicate(predicate: Predicate): HiveSource

def withProjection(columns: Seq[String]): HiveSource

def withProjection(first: String, rest: String*): HiveSource

Deprecated Value Members

def select(first: String, rest: String*): HiveSource

Inherited from Serializable

Inherited from Serializable

Inherited from Product

Inherited from Equals

Inherited from Using

Inherited from Source

Inherited from Logging

Inherited from AnyRef

Inherited from Any

Ungrouped