trait SparkStreamlet extends Streamlet[SparkStreamletContext] with Serializable
The base class for defining Spark streamlets. Derived classes need to override createLogic
to
provide the custom implementation for the behavior of the streamlet.
Here's an example:
// new custom `SparkStreamlet` object MySparkProcessor extends SparkStreamlet { // Step 1: Define inlets and outlets. Note for the outlet you need to specify // the partitioner function explicitly val in = AvroInlet[Data]("in") val out = AvroOutlet[Simple]("out", _.name) // Step 2: Define the shape of the streamlet. In this example the streamlet // has 1 inlet and 1 outlet val shape = StreamletShape(in, out) // Step 3: Provide custom implementation of `SparkStreamletLogic` that defines // the behavior of the streamlet override def createLogic() = new SparkStreamletLogic { override def buildStreamingQueries = { val dataset = readStream(in) val outStream = dataset.select($"name").as[Simple] val query = writeStream(outStream, out, OutputMode.Append) Seq(query) } } }
Linear Supertypes
Ordering
- Alphabetic
- By Inheritance
Inherited
- SparkStreamlet
- Serializable
- Serializable
- Streamlet
- AnyRef
- Any
- Hide All
- Show All
Visibility
- Public
- All
Abstract Value Members
-
abstract
def
createLogic(): SparkStreamletLogic
- Attributes
- protected
-
abstract
def
shape(): StreamletShape
- Definition Classes
- Streamlet
Concrete Value Members
-
final
def
!=(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
final
def
##(): Int
- Definition Classes
- AnyRef → Any
-
final
def
==(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
- val StopTimeout: FiniteDuration
- val applicationName: String
-
final
def
asInstanceOf[T0]: T0
- Definition Classes
- Any
-
def
attributes: Set[StreamletAttribute]
- Attributes
- protected[cloudflow]
- Definition Classes
- Streamlet
-
def
clone(): AnyRef
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()
-
def
configParameters: IndexedSeq[ConfigParameter]
- Definition Classes
- Streamlet
- final def configuredValue(context: SparkStreamletContext, configKey: String): String
-
implicit final
def
context: SparkStreamletContext
- Attributes
- protected
- Definition Classes
- Streamlet
-
final
def
createContext(config: Config): SparkStreamletContext
- Attributes
- protected
- Definition Classes
- SparkStreamlet → Streamlet
-
def
customAttributes: Set[StreamletAttribute]
- Definition Classes
- Streamlet
-
def
defineConfigParameters(): Array[ConfigParameter]
- Definition Classes
- Streamlet
-
def
defineCustomAttributes(): Array[StreamletAttribute]
- Definition Classes
- Streamlet
-
def
defineVolumeMounts(): Array[VolumeMount]
- Definition Classes
- Streamlet
-
def
description: String
- Definition Classes
- Streamlet
-
final
def
eq(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
def
equals(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
def
finalize(): Unit
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( classOf[java.lang.Throwable] )
-
final
def
getClass(): Class[_]
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
-
final
def
getContext(): SparkStreamletContext
- Attributes
- protected
- Definition Classes
- Streamlet
-
def
hashCode(): Int
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
-
final
def
inlets: IndexedSeq[Inlet]
- Definition Classes
- Streamlet
-
final
def
isInstanceOf[T0]: Boolean
- Definition Classes
- Any
-
final
def
jsonDescriptor: String
- Definition Classes
- Streamlet
-
def
labels: IndexedSeq[String]
- Definition Classes
- Streamlet
-
lazy val
log: Logger
- Attributes
- protected
- Definition Classes
- Streamlet
- Annotations
- @transient()
-
def
logStartRunnerMessage(buildInfo: String): Unit
- Definition Classes
- SparkStreamlet → Streamlet
-
final
def
ne(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
final
def
notify(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
-
final
def
notifyAll(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
-
final
def
outlets: IndexedSeq[Outlet]
- Definition Classes
- Streamlet
-
final
def
run(context: SparkStreamletContext): StreamletExecution
- Definition Classes
- SparkStreamlet → Streamlet
-
final
val
runtime: SparkStreamletRuntime.type
- Definition Classes
- SparkStreamlet → Streamlet
-
final
def
synchronized[T0](arg0: ⇒ T0): T0
- Definition Classes
- AnyRef
-
def
toString(): String
- Definition Classes
- AnyRef → Any
-
def
volumeMounts: IndexedSeq[VolumeMount]
- Definition Classes
- Streamlet
-
final
def
wait(): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long, arg1: Int): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()