KafkaWriter

Class used to write DStreams, RDDs and Datasets to Kafka

Example usage:

import com.github.benfradet.spark.kafka.writer.KafkaWriter._
import org.apache.kafka.common.serialization.StringSerializer

val topic = "my-topic"
val producerConfig = Map(
  "bootstrap.servers" -> "127.0.0.1:9092",
  "key.serializer" -> classOf[StringSerializer].getName,
  "value.serializer" -> classOf[StringSerializer].getName
)

val dStream: DStream[String] = ...
dStream.writeToKafka(
  producerConfig,
  s => new ProducerRecord[String, String](topic, s)
)

val rdd: RDD[String] = ...
rdd.writeToKafka(
  producerConfig,
  s => new ProducerRecord[String, String](localTopic, s)
)

val dataset: Dataset[Foo] = ...
dataset.writeToKafka(
  producerConfig,
  f => new ProducerRecord[String, String](localTopic, f.toString)
)

val dataFrame: DataFrame = ...
dataFrame.writeToKafka(
  producerConfig,
  r => new ProducerRecord[String, String](localTopic, r.get(0).toString)
)

It is also possible to provide a callback for each write to Kafka.

This is optional and has a value of None by default.

Example Usage:

@transient val log = org.apache.log4j.Logger.getLogger("spark-kafka-writer")

val dStream: DStream[String] = ...
dStream.writeToKafka(
  producerConfig,
  s => new ProducerRecord[String, String](topic, s),
  Some(new Callback with Serializable {
    override def onCompletion(metadata: RecordMetadata, e: Exception): Unit = {
      if (Option(e).isDefined) {
        log.warn("error sending message", e)
      } else {
        log.info(s"write succeeded, offset: ${metadata.offset()")
      }
    }
  })
)

Linear Supertypes

Serializable, Serializable, AnyRef, Any

Known Subclasses

DStreamKafkaWriter, DatasetKafkaWriter, RDDKafkaWriter

Instance Constructors

new KafkaWriter()(implicit arg0: ClassTag[T])

Abstract Value Members

abstract def writeToKafka[K, V](producerConfig: Map[String, AnyRef], transformFunc: (T) ⇒ ProducerRecord[K, V], callback: Option[Callback] = None): Unit

Write a DStream, RDD, or Dataset to Kafka
Write a DStream, RDD, or Dataset to Kafka
producerConfig
producer configuration for creating KafkaProducer
transformFunc
a function used to transform values of T type into ProducerRecords
callback
an optional Callback to be called after each write, default value is None.

Concrete Value Members

final def !=(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def !=(arg0: Any): Boolean

Definition Classes
Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def ==(arg0: Any): Boolean

Definition Classes
Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def hashCode(): Int

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

abstract class KafkaWriter[T] extends Serializable

Instance Constructors

new KafkaWriter()(implicit arg0: ClassTag[T])

Abstract Value Members

abstract def writeToKafka[K, V](producerConfig: Map[String, AnyRef], transformFunc: (T) ⇒ ProducerRecord[K, V], callback: Option[Callback] = None): Unit

Concrete Value Members

final def !=(arg0: AnyRef): Boolean

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: AnyRef): Boolean

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def clone(): AnyRef

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def getClass(): Class[_]

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from Serializable

Inherited from Serializable

Inherited from AnyRef

Inherited from Any

Ungrouped