DirectDatahubInputDStream

Instance Constructors

new DirectDatahubInputDStream(_ssc: StreamingContext, endpoint: String, project: String, topic: String, subId: String, accessId: String, accessKey: String, func: (RecordEntry) ⇒ String, mode: CursorType, zkParam: Map[String, String])

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
val baseScope: Option[String]

Attributes
protected[org.apache.spark.streaming]
Definition Classes
InputDStream → DStream
def cache(): DStream[Array[Byte]]

Definition Classes
DStream
def checkpoint(interval: Duration): DStream[Array[Byte]]

Definition Classes
DStream
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def commitAsync(): Unit

:: Experimental :: Queue up offset ranges for commit to Loghub at a future time.
:: Experimental :: Queue up offset ranges for commit to Loghub at a future time. Threadsafe. This is only needed if you intend to store offsets in Loghub, instead of your own store.

Definition Classes
DirectDatahubInputDStream → CanCommitOffsets
def compute(validTime: Time): Option[RDD[Array[Byte]]]

Definition Classes
DirectDatahubInputDStream → DStream
def context: StreamingContext

Definition Classes
DStream
def count(): DStream[Long]

Definition Classes
DStream
def countByValue(numPartitions: Int)(implicit ord: Ordering[Array[Byte]]): DStream[(Array[Byte], Long)]

Definition Classes
DStream
def countByValueAndWindow(windowDuration: Duration, slideDuration: Duration, numPartitions: Int)(implicit ord: Ordering[Array[Byte]]): DStream[(Array[Byte], Long)]

Definition Classes
DStream
def countByWindow(windowDuration: Duration, slideDuration: Duration): DStream[Long]

Definition Classes
DStream
def createRDDWithLocalProperties[U](time: Time, displayInnerRDDOps: Boolean)(body: ⇒ U): U

Attributes
protected[org.apache.spark.streaming]
Definition Classes
DStream
var datahubCommitPathRoot: String
var datahubConsumePathRoot: String
def dependencies: List[DStream[_]]

Definition Classes
InputDStream → DStream
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def filter(filterFunc: (Array[Byte]) ⇒ Boolean): DStream[Array[Byte]]

Definition Classes
DStream
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
def flatMap[U](flatMapFunc: (Array[Byte]) ⇒ TraversableOnce[U])(implicit arg0: ClassTag[U]): DStream[U]

Definition Classes
DStream
def foreachRDD(foreachFunc: (RDD[Array[Byte]], Time) ⇒ Unit): Unit

Definition Classes
DStream
def foreachRDD(foreachFunc: (RDD[Array[Byte]]) ⇒ Unit): Unit

Definition Classes
DStream
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def glom(): DStream[Array[Array[Byte]]]

Definition Classes
DStream
def hashCode(): Int

Definition Classes
AnyRef → Any
val id: Int

Definition Classes
InputDStream
def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean

Attributes
protected
Definition Classes
Logging
def initializeLogIfNecessary(isInterpreter: Boolean): Unit

Attributes
protected
Definition Classes
Logging
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def isTraceEnabled(): Boolean

Attributes
protected
Definition Classes
Logging
def log: Logger

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logName: String

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def map[U](mapFunc: (Array[Byte]) ⇒ U)(implicit arg0: ClassTag[U]): DStream[U]

Definition Classes
DStream
def mapPartitions[U](mapPartFunc: (Iterator[Array[Byte]]) ⇒ Iterator[U], preservePartitioning: Boolean)(implicit arg0: ClassTag[U]): DStream[U]

Definition Classes
DStream
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def persist(): DStream[Array[Byte]]

Definition Classes
DStream
def persist(level: StorageLevel): DStream[Array[Byte]]

Definition Classes
DStream
def print(num: Int): Unit

Definition Classes
DStream
def print(): Unit

Definition Classes
DStream
val rateController: Option[RateController]

Attributes
protected[org.apache.spark.streaming]
Definition Classes
InputDStream
def reduce(reduceFunc: (Array[Byte], Array[Byte]) ⇒ Array[Byte]): DStream[Array[Byte]]

Definition Classes
DStream
def reduceByWindow(reduceFunc: (Array[Byte], Array[Byte]) ⇒ Array[Byte], invReduceFunc: (Array[Byte], Array[Byte]) ⇒ Array[Byte], windowDuration: Duration, slideDuration: Duration): DStream[Array[Byte]]

Definition Classes
DStream
def reduceByWindow(reduceFunc: (Array[Byte], Array[Byte]) ⇒ Array[Byte], windowDuration: Duration, slideDuration: Duration): DStream[Array[Byte]]

Definition Classes
DStream
def repartition(numPartitions: Int): DStream[Array[Byte]]

Definition Classes
DStream
def saveAsObjectFiles(prefix: String, suffix: String): Unit

Definition Classes
DStream
def saveAsTextFiles(prefix: String, suffix: String): Unit

Definition Classes
DStream
def slice(fromTime: Time, toTime: Time): Seq[RDD[Array[Byte]]]

Definition Classes
DStream
def slice(interval: Interval): Seq[RDD[Array[Byte]]]

Definition Classes
DStream
def slideDuration: Duration

Definition Classes
InputDStream → DStream
def start(): Unit

Definition Classes
DirectDatahubInputDStream → InputDStream
def stop(): Unit

Definition Classes
DirectDatahubInputDStream → InputDStream
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
def transform[U](transformFunc: (RDD[Array[Byte]], Time) ⇒ RDD[U])(implicit arg0: ClassTag[U]): DStream[U]

Definition Classes
DStream
def transform[U](transformFunc: (RDD[Array[Byte]]) ⇒ RDD[U])(implicit arg0: ClassTag[U]): DStream[U]

Definition Classes
DStream
def transformWith[U, V](other: DStream[U], transformFunc: (RDD[Array[Byte]], RDD[U], Time) ⇒ RDD[V])(implicit arg0: ClassTag[U], arg1: ClassTag[V]): DStream[V]

Definition Classes
DStream
def transformWith[U, V](other: DStream[U], transformFunc: (RDD[Array[Byte]], RDD[U]) ⇒ RDD[V])(implicit arg0: ClassTag[U], arg1: ClassTag[V]): DStream[V]

Definition Classes
DStream
def union(that: DStream[Array[Byte]]): DStream[Array[Byte]]

Definition Classes
DStream
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
def window(windowDuration: Duration, slideDuration: Duration): DStream[Array[Byte]]

Definition Classes
DStream
def window(windowDuration: Duration): DStream[Array[Byte]]

Definition Classes
DStream

Related Docs: object DirectDatahubInputDStream | package datahub

class DirectDatahubInputDStream extends InputDStream[Array[Byte]] with CanCommitOffsets

Instance Constructors

new DirectDatahubInputDStream(_ssc: StreamingContext, endpoint: String, project: String, topic: String, subId: String, accessId: String, accessKey: String, func: (RecordEntry) ⇒ String, mode: CursorType, zkParam: Map[String, String])

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

val baseScope: Option[String]

def cache(): DStream[Array[Byte]]

def checkpoint(interval: Duration): DStream[Array[Byte]]

def clone(): AnyRef

def commitAsync(): Unit

def compute(validTime: Time): Option[RDD[Array[Byte]]]

def context: StreamingContext

def count(): DStream[Long]

def countByValue(numPartitions: Int)(implicit ord: Ordering[Array[Byte]]): DStream[(Array[Byte], Long)]

def countByValueAndWindow(windowDuration: Duration, slideDuration: Duration, numPartitions: Int)(implicit ord: Ordering[Array[Byte]]): DStream[(Array[Byte], Long)]

def countByWindow(windowDuration: Duration, slideDuration: Duration): DStream[Long]

def createRDDWithLocalProperties[U](time: Time, displayInnerRDDOps: Boolean)(body: ⇒ U): U

var datahubCommitPathRoot: String

var datahubConsumePathRoot: String

def dependencies: List[DStream[_]]

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def filter(filterFunc: (Array[Byte]) ⇒ Boolean): DStream[Array[Byte]]

def finalize(): Unit

def flatMap[U](flatMapFunc: (Array[Byte]) ⇒ TraversableOnce[U])(implicit arg0: ClassTag[U]): DStream[U]

def foreachRDD(foreachFunc: (RDD[Array[Byte]], Time) ⇒ Unit): Unit

def foreachRDD(foreachFunc: (RDD[Array[Byte]]) ⇒ Unit): Unit

final def getClass(): Class[_]

def glom(): DStream[Array[Array[Byte]]]

def hashCode(): Int

val id: Int

def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean

def initializeLogIfNecessary(isInterpreter: Boolean): Unit

final def isInstanceOf[T0]: Boolean

def isTraceEnabled(): Boolean

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logName: String

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

def map[U](mapFunc: (Array[Byte]) ⇒ U)(implicit arg0: ClassTag[U]): DStream[U]

def mapPartitions[U](mapPartFunc: (Iterator[Array[Byte]]) ⇒ Iterator[U], preservePartitioning: Boolean)(implicit arg0: ClassTag[U]): DStream[U]

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def persist(): DStream[Array[Byte]]

def persist(level: StorageLevel): DStream[Array[Byte]]

def print(num: Int): Unit

def print(): Unit

val rateController: Option[RateController]

def reduce(reduceFunc: (Array[Byte], Array[Byte]) ⇒ Array[Byte]): DStream[Array[Byte]]

def reduceByWindow(reduceFunc: (Array[Byte], Array[Byte]) ⇒ Array[Byte], invReduceFunc: (Array[Byte], Array[Byte]) ⇒ Array[Byte], windowDuration: Duration, slideDuration: Duration): DStream[Array[Byte]]

def reduceByWindow(reduceFunc: (Array[Byte], Array[Byte]) ⇒ Array[Byte], windowDuration: Duration, slideDuration: Duration): DStream[Array[Byte]]

def repartition(numPartitions: Int): DStream[Array[Byte]]

def saveAsObjectFiles(prefix: String, suffix: String): Unit

def saveAsTextFiles(prefix: String, suffix: String): Unit

def slice(fromTime: Time, toTime: Time): Seq[RDD[Array[Byte]]]

def slice(interval: Interval): Seq[RDD[Array[Byte]]]

def slideDuration: Duration

def start(): Unit

def stop(): Unit

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

def transform[U](transformFunc: (RDD[Array[Byte]], Time) ⇒ RDD[U])(implicit arg0: ClassTag[U]): DStream[U]

def transform[U](transformFunc: (RDD[Array[Byte]]) ⇒ RDD[U])(implicit arg0: ClassTag[U]): DStream[U]

def transformWith[U, V](other: DStream[U], transformFunc: (RDD[Array[Byte]], RDD[U], Time) ⇒ RDD[V])(implicit arg0: ClassTag[U], arg1: ClassTag[V]): DStream[V]

def transformWith[U, V](other: DStream[U], transformFunc: (RDD[Array[Byte]], RDD[U]) ⇒ RDD[V])(implicit arg0: ClassTag[U], arg1: ClassTag[V]): DStream[V]

def union(that: DStream[Array[Byte]]): DStream[Array[Byte]]