TestStreamingContext

Instance Constructors

new TestStreamingContext(sparkContext: SparkContext, batchDuration: Duration)

Create a TestableStreamingContext using an existing SparkContext.
Create a TestableStreamingContext using an existing SparkContext.
sparkContext
existing SparkContext
batchDuration
the time interval at which streaming data will be divided into batches
new TestStreamingContext(sc_: SparkContext, cp_: Checkpoint, batchDur_: Duration)

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def addStreamingListener(streamingListener: StreamingListener): Unit

Definition Classes
StreamingContext
final def asInstanceOf[T0]: T0

Definition Classes
Any
def awaitTermination(): Unit

Definition Classes
StreamingContext
def awaitTerminationOrTimeout(timeout: Long): Boolean

Definition Classes
StreamingContext
def binaryRecordsStream(directory: String, recordLength: Int): DStream[Array[Byte]]

Definition Classes
StreamingContext
def checkpoint(directory: String): Unit

Definition Classes
StreamingContext
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@HotSpotIntrinsicCandidate() @throws( ... )
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def fileStream[K, V, F <: InputFormat[K, V]](directory: String, filter: (Path) ⇒ Boolean, newFilesOnly: Boolean, conf: Configuration)(implicit arg0: ClassTag[K], arg1: ClassTag[V], arg2: ClassTag[F]): InputDStream[(K, V)]

Definition Classes
StreamingContext
def fileStream[K, V, F <: InputFormat[K, V]](directory: String, filter: (Path) ⇒ Boolean, newFilesOnly: Boolean)(implicit arg0: ClassTag[K], arg1: ClassTag[V], arg2: ClassTag[F]): InputDStream[(K, V)]

Definition Classes
StreamingContext
def fileStream[K, V, F <: InputFormat[K, V]](directory: String)(implicit arg0: ClassTag[K], arg1: ClassTag[V], arg2: ClassTag[F]): InputDStream[(K, V)]

Definition Classes
StreamingContext
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
Annotations
@HotSpotIntrinsicCandidate()
def getScheduler(): JobScheduler
def getState(): StreamingContextState

Definition Classes
StreamingContext
Annotations
@DeveloperApi()
def hashCode(): Int

Definition Classes
AnyRef → Any
Annotations
@HotSpotIntrinsicCandidate()
def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean

Attributes
protected
Definition Classes
Logging
def initializeLogIfNecessary(isInterpreter: Boolean): Unit

Attributes
protected
Definition Classes
Logging
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def isTraceEnabled(): Boolean

Attributes
protected
Definition Classes
Logging
def log: Logger

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logName: String

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
Annotations
@HotSpotIntrinsicCandidate()
final def notifyAll(): Unit

Definition Classes
AnyRef
Annotations
@HotSpotIntrinsicCandidate()
def queueStream[T](queue: Queue[RDD[T]], oneAtATime: Boolean, defaultRDD: RDD[T])(implicit arg0: ClassTag[T]): InputDStream[T]

Definition Classes
StreamingContext
def queueStream[T](queue: Queue[RDD[T]], oneAtATime: Boolean)(implicit arg0: ClassTag[T]): InputDStream[T]

Definition Classes
StreamingContext
def rawSocketStream[T](hostname: String, port: Int, storageLevel: StorageLevel)(implicit arg0: ClassTag[T]): ReceiverInputDStream[T]

Definition Classes
StreamingContext
def receiverStream[T](receiver: Receiver[T])(implicit arg0: ClassTag[T]): ReceiverInputDStream[T]

Definition Classes
StreamingContext
def remember(duration: Duration): Unit

Definition Classes
StreamingContext
def removeStreamingListener(streamingListener: StreamingListener): Unit

Definition Classes
StreamingContext
def socketStream[T](hostname: String, port: Int, converter: (InputStream) ⇒ Iterator[T], storageLevel: StorageLevel)(implicit arg0: ClassTag[T]): ReceiverInputDStream[T]

Definition Classes
StreamingContext
def socketTextStream(hostname: String, port: Int, storageLevel: StorageLevel): ReceiverInputDStream[String]

Definition Classes
StreamingContext
def sparkContext: SparkContext

Definition Classes
StreamingContext
def start(): Unit

Definition Classes
StreamingContext
def stop(stopSparkContext: Boolean, stopGracefully: Boolean): Unit

Definition Classes
StreamingContext
def stop(stopSparkContext: Boolean): Unit

Definition Classes
StreamingContext
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def textFileStream(directory: String): DStream[String]

Definition Classes
StreamingContext
def toString(): String

Definition Classes
AnyRef → Any
def transform[T](dstreams: Seq[DStream[_]], transformFunc: (Seq[RDD[_]], Time) ⇒ RDD[T])(implicit arg0: ClassTag[T]): DStream[T]

Definition Classes
StreamingContext
def union[T](streams: Seq[DStream[T]])(implicit arg0: ClassTag[T]): DStream[T]

Definition Classes
StreamingContext
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Deprecated Value Members

def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@Deprecated @deprecated @throws( classOf[java.lang.Throwable] )
Deprecated
(Since version ) see corresponding Javadoc for more information.

Related Doc: package streaming

class TestStreamingContext extends StreamingContext

Instance Constructors

new TestStreamingContext(sparkContext: SparkContext, batchDuration: Duration)

new TestStreamingContext(sc_: SparkContext, cp_: Checkpoint, batchDur_: Duration)

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def addStreamingListener(streamingListener: StreamingListener): Unit

final def asInstanceOf[T0]: T0

def awaitTermination(): Unit

def awaitTerminationOrTimeout(timeout: Long): Boolean

def binaryRecordsStream(directory: String, recordLength: Int): DStream[Array[Byte]]

def checkpoint(directory: String): Unit

def clone(): AnyRef

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def fileStream[K, V, F <: InputFormat[K, V]](directory: String, filter: (Path) ⇒ Boolean, newFilesOnly: Boolean, conf: Configuration)(implicit arg0: ClassTag[K], arg1: ClassTag[V], arg2: ClassTag[F]): InputDStream[(K, V)]

def fileStream[K, V, F <: InputFormat[K, V]](directory: String, filter: (Path) ⇒ Boolean, newFilesOnly: Boolean)(implicit arg0: ClassTag[K], arg1: ClassTag[V], arg2: ClassTag[F]): InputDStream[(K, V)]

def fileStream[K, V, F <: InputFormat[K, V]](directory: String)(implicit arg0: ClassTag[K], arg1: ClassTag[V], arg2: ClassTag[F]): InputDStream[(K, V)]

final def getClass(): Class[_]

def getScheduler(): JobScheduler

def getState(): StreamingContextState

def hashCode(): Int

def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean

def initializeLogIfNecessary(isInterpreter: Boolean): Unit

final def isInstanceOf[T0]: Boolean

def isTraceEnabled(): Boolean

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logName: String

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def queueStream[T](queue: Queue[RDD[T]], oneAtATime: Boolean, defaultRDD: RDD[T])(implicit arg0: ClassTag[T]): InputDStream[T]

def queueStream[T](queue: Queue[RDD[T]], oneAtATime: Boolean)(implicit arg0: ClassTag[T]): InputDStream[T]

def rawSocketStream[T](hostname: String, port: Int, storageLevel: StorageLevel)(implicit arg0: ClassTag[T]): ReceiverInputDStream[T]

def receiverStream[T](receiver: Receiver[T])(implicit arg0: ClassTag[T]): ReceiverInputDStream[T]

def remember(duration: Duration): Unit

def removeStreamingListener(streamingListener: StreamingListener): Unit

def socketStream[T](hostname: String, port: Int, converter: (InputStream) ⇒ Iterator[T], storageLevel: StorageLevel)(implicit arg0: ClassTag[T]): ReceiverInputDStream[T]

def socketTextStream(hostname: String, port: Int, storageLevel: StorageLevel): ReceiverInputDStream[String]

def sparkContext: SparkContext

def start(): Unit

def stop(stopSparkContext: Boolean, stopGracefully: Boolean): Unit

def stop(stopSparkContext: Boolean): Unit

final def synchronized[T0](arg0: ⇒ T0): T0

def textFileStream(directory: String): DStream[String]

def toString(): String

def transform[T](dstreams: Seq[DStream[_]], transformFunc: (Seq[RDD[_]], Time) ⇒ RDD[T])(implicit arg0: ClassTag[T]): DStream[T]

def union[T](streams: Seq[DStream[T]])(implicit arg0: ClassTag[T]): DStream[T]

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

final def wait(): Unit

Deprecated Value Members

def finalize(): Unit

Inherited from StreamingContext

Inherited from internal.Logging

Inherited from AnyRef

Inherited from Any

Ungrouped