Class

eu.shiftforward.adstax.spark

AdStaxSparkContext

Related Doc: package spark

Permalink

class AdStaxSparkContext extends SparkContext

A context for running spark jobs which provides an extra method to access AdStax events.

Linear Supertypes
SparkContext, Logging, AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. AdStaxSparkContext
  2. SparkContext
  3. Logging
  4. AnyRef
  5. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. All

Instance Constructors

  1. new AdStaxSparkContext(conf: SparkConf, esIndex: String = "http-*")

    Permalink

    conf

    the Spark configuration to use when initializing the context

    esIndex

    the elasticsearch index where to search for events

Value Members

  1. final def !=(arg0: Any): Boolean

    Permalink
    Definition Classes
    AnyRef → Any
  2. final def ##(): Int

    Permalink
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean

    Permalink
    Definition Classes
    AnyRef → Any
  4. def addFile(path: String, recursive: Boolean): Unit

    Permalink
    Definition Classes
    SparkContext
  5. def addFile(path: String): Unit

    Permalink
    Definition Classes
    SparkContext
  6. def addJar(path: String): Unit

    Permalink
    Definition Classes
    SparkContext
  7. def addSparkListener(listener: SparkListenerInterface): Unit

    Permalink
    Definition Classes
    SparkContext
    Annotations
    @DeveloperApi()
  8. def appName: String

    Permalink
    Definition Classes
    SparkContext
  9. def applicationAttemptId: Option[String]

    Permalink
    Definition Classes
    SparkContext
  10. def applicationId: String

    Permalink
    Definition Classes
    SparkContext
  11. final def asInstanceOf[T0]: T0

    Permalink
    Definition Classes
    Any
  12. def binaryFiles(path: String, minPartitions: Int): RDD[(String, PortableDataStream)]

    Permalink
    Definition Classes
    SparkContext
  13. def binaryRecords(path: String, recordLength: Int, conf: Configuration): RDD[Array[Byte]]

    Permalink
    Definition Classes
    SparkContext
  14. def broadcast[T](value: T)(implicit arg0: ClassTag[T]): Broadcast[T]

    Permalink
    Definition Classes
    SparkContext
  15. def cancelAllJobs(): Unit

    Permalink
    Definition Classes
    SparkContext
  16. def cancelJob(jobId: Int): Unit

    Permalink
    Definition Classes
    SparkContext
  17. def cancelJob(jobId: Int, reason: String): Unit

    Permalink
    Definition Classes
    SparkContext
  18. def cancelJobGroup(groupId: String): Unit

    Permalink
    Definition Classes
    SparkContext
  19. def cancelStage(stageId: Int): Unit

    Permalink
    Definition Classes
    SparkContext
  20. def cancelStage(stageId: Int, reason: String): Unit

    Permalink
    Definition Classes
    SparkContext
  21. def checkpointFile[T](path: String)(implicit arg0: ClassTag[T]): RDD[T]

    Permalink
    Attributes
    protected[org.apache.spark]
    Definition Classes
    SparkContext
  22. def clearCallSite(): Unit

    Permalink
    Definition Classes
    SparkContext
  23. def clearJobGroup(): Unit

    Permalink
    Definition Classes
    SparkContext
  24. def clone(): AnyRef

    Permalink
    Attributes
    protected[java.lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  25. def collectionAccumulator[T](name: String): CollectionAccumulator[T]

    Permalink
    Definition Classes
    SparkContext
  26. def collectionAccumulator[T]: CollectionAccumulator[T]

    Permalink
    Definition Classes
    SparkContext
  27. def defaultMinPartitions: Int

    Permalink
    Definition Classes
    SparkContext
  28. def defaultParallelism: Int

    Permalink
    Definition Classes
    SparkContext
  29. def deployMode: String

    Permalink
    Definition Classes
    SparkContext
  30. def doubleAccumulator(name: String): DoubleAccumulator

    Permalink
    Definition Classes
    SparkContext
  31. def doubleAccumulator: DoubleAccumulator

    Permalink
    Definition Classes
    SparkContext
  32. def emptyRDD[T](implicit arg0: ClassTag[T]): RDD[T]

    Permalink
    Definition Classes
    SparkContext
  33. final def eq(arg0: AnyRef): Boolean

    Permalink
    Definition Classes
    AnyRef
  34. def equals(arg0: Any): Boolean

    Permalink
    Definition Classes
    AnyRef → Any
  35. def eventsRDD(eventTypes: Set[String], startDateTime: com.github.nscala_time.time.Imports.DateTime, endDateTime: com.github.nscala_time.time.Imports.DateTime): RDD[String]

    Permalink

    Returns all AdStax events between startDateTime and endDateTime which have a type contained in eventTypes.

    Returns all AdStax events between startDateTime and endDateTime which have a type contained in eventTypes. In case eventTypes is empty, all event types are considered.

    eventTypes

    the type of events to fetch. If empty, all event types are considered

    startDateTime

    the initial timestamp of the events to fetch

    endDateTime

    the final timestamp of the events to fetch

    returns

    a RDD of Strings with the JSON representation of the fetched events.

  36. def files: Seq[String]

    Permalink
    Definition Classes
    SparkContext
  37. def finalize(): Unit

    Permalink
    Attributes
    protected[java.lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] )
  38. def getAllPools: Seq[Schedulable]

    Permalink
    Definition Classes
    SparkContext
    Annotations
    @DeveloperApi()
  39. def getCheckpointDir: Option[String]

    Permalink
    Definition Classes
    SparkContext
  40. final def getClass(): Class[_]

    Permalink
    Definition Classes
    AnyRef → Any
  41. def getConf: SparkConf

    Permalink
    Definition Classes
    SparkContext
  42. def getExecutorMemoryStatus: Map[String, (Long, Long)]

    Permalink
    Definition Classes
    SparkContext
  43. def getLocalProperty(key: String): String

    Permalink
    Definition Classes
    SparkContext
  44. def getPersistentRDDs: Map[Int, RDD[_]]

    Permalink
    Definition Classes
    SparkContext
  45. def getPoolForName(pool: String): Option[Schedulable]

    Permalink
    Definition Classes
    SparkContext
    Annotations
    @DeveloperApi()
  46. def getRDDStorageInfo: Array[RDDInfo]

    Permalink
    Definition Classes
    SparkContext
    Annotations
    @DeveloperApi()
  47. def getSchedulingMode: SchedulingMode

    Permalink
    Definition Classes
    SparkContext
  48. def hadoopConfiguration: Configuration

    Permalink
    Definition Classes
    SparkContext
  49. def hadoopFile[K, V, F <: InputFormat[K, V]](path: String)(implicit km: ClassTag[K], vm: ClassTag[V], fm: ClassTag[F]): RDD[(K, V)]

    Permalink
    Definition Classes
    SparkContext
  50. def hadoopFile[K, V, F <: InputFormat[K, V]](path: String, minPartitions: Int)(implicit km: ClassTag[K], vm: ClassTag[V], fm: ClassTag[F]): RDD[(K, V)]

    Permalink
    Definition Classes
    SparkContext
  51. def hadoopFile[K, V](path: String, inputFormatClass: Class[_ <: InputFormat[K, V]], keyClass: Class[K], valueClass: Class[V], minPartitions: Int): RDD[(K, V)]

    Permalink
    Definition Classes
    SparkContext
  52. def hadoopRDD[K, V](conf: JobConf, inputFormatClass: Class[_ <: InputFormat[K, V]], keyClass: Class[K], valueClass: Class[V], minPartitions: Int): RDD[(K, V)]

    Permalink
    Definition Classes
    SparkContext
  53. def hashCode(): Int

    Permalink
    Definition Classes
    AnyRef → Any
  54. def initializeLogIfNecessary(isInterpreter: Boolean): Unit

    Permalink
    Attributes
    protected
    Definition Classes
    Logging
  55. final def isInstanceOf[T0]: Boolean

    Permalink
    Definition Classes
    Any
  56. def isLocal: Boolean

    Permalink
    Definition Classes
    SparkContext
  57. def isStopped: Boolean

    Permalink
    Definition Classes
    SparkContext
  58. def isTraceEnabled(): Boolean

    Permalink
    Attributes
    protected
    Definition Classes
    Logging
  59. def jars: Seq[String]

    Permalink
    Definition Classes
    SparkContext
  60. def killExecutor(executorId: String): Boolean

    Permalink
    Definition Classes
    SparkContext
    Annotations
    @DeveloperApi()
  61. def killExecutors(executorIds: Seq[String]): Boolean

    Permalink
    Definition Classes
    SparkContext
    Annotations
    @DeveloperApi()
  62. def killTaskAttempt(taskId: Long, interruptThread: Boolean, reason: String): Boolean

    Permalink
    Definition Classes
    SparkContext
  63. def listFiles(): Seq[String]

    Permalink
    Definition Classes
    SparkContext
  64. def listJars(): Seq[String]

    Permalink
    Definition Classes
    SparkContext
  65. val localProperties: InheritableThreadLocal[Properties]

    Permalink
    Attributes
    protected[org.apache.spark]
    Definition Classes
    SparkContext
  66. def log: Logger

    Permalink
    Attributes
    protected
    Definition Classes
    Logging
  67. def logDebug(msg: ⇒ String, throwable: Throwable): Unit

    Permalink
    Attributes
    protected
    Definition Classes
    Logging
  68. def logDebug(msg: ⇒ String): Unit

    Permalink
    Attributes
    protected
    Definition Classes
    Logging
  69. def logError(msg: ⇒ String, throwable: Throwable): Unit

    Permalink
    Attributes
    protected
    Definition Classes
    Logging
  70. def logError(msg: ⇒ String): Unit

    Permalink
    Attributes
    protected
    Definition Classes
    Logging
  71. def logInfo(msg: ⇒ String, throwable: Throwable): Unit

    Permalink
    Attributes
    protected
    Definition Classes
    Logging
  72. def logInfo(msg: ⇒ String): Unit

    Permalink
    Attributes
    protected
    Definition Classes
    Logging
  73. def logName: String

    Permalink
    Attributes
    protected
    Definition Classes
    Logging
  74. def logTrace(msg: ⇒ String, throwable: Throwable): Unit

    Permalink
    Attributes
    protected
    Definition Classes
    Logging
  75. def logTrace(msg: ⇒ String): Unit

    Permalink
    Attributes
    protected
    Definition Classes
    Logging
  76. def logWarning(msg: ⇒ String, throwable: Throwable): Unit

    Permalink
    Attributes
    protected
    Definition Classes
    Logging
  77. def logWarning(msg: ⇒ String): Unit

    Permalink
    Attributes
    protected
    Definition Classes
    Logging
  78. def longAccumulator(name: String): LongAccumulator

    Permalink
    Definition Classes
    SparkContext
  79. def longAccumulator: LongAccumulator

    Permalink
    Definition Classes
    SparkContext
  80. def makeRDD[T](seq: Seq[(T, Seq[String])])(implicit arg0: ClassTag[T]): RDD[T]

    Permalink
    Definition Classes
    SparkContext
  81. def makeRDD[T](seq: Seq[T], numSlices: Int)(implicit arg0: ClassTag[T]): RDD[T]

    Permalink
    Definition Classes
    SparkContext
  82. def master: String

    Permalink
    Definition Classes
    SparkContext
  83. final def ne(arg0: AnyRef): Boolean

    Permalink
    Definition Classes
    AnyRef
  84. def newAPIHadoopFile[K, V, F <: InputFormat[K, V]](path: String, fClass: Class[F], kClass: Class[K], vClass: Class[V], conf: Configuration): RDD[(K, V)]

    Permalink
    Definition Classes
    SparkContext
  85. def newAPIHadoopFile[K, V, F <: InputFormat[K, V]](path: String)(implicit km: ClassTag[K], vm: ClassTag[V], fm: ClassTag[F]): RDD[(K, V)]

    Permalink
    Definition Classes
    SparkContext
  86. def newAPIHadoopRDD[K, V, F <: InputFormat[K, V]](conf: Configuration, fClass: Class[F], kClass: Class[K], vClass: Class[V]): RDD[(K, V)]

    Permalink
    Definition Classes
    SparkContext
  87. final def notify(): Unit

    Permalink
    Definition Classes
    AnyRef
  88. final def notifyAll(): Unit

    Permalink
    Definition Classes
    AnyRef
  89. def objectFile[T](path: String, minPartitions: Int)(implicit arg0: ClassTag[T]): RDD[T]

    Permalink
    Definition Classes
    SparkContext
  90. def parallelize[T](seq: Seq[T], numSlices: Int)(implicit arg0: ClassTag[T]): RDD[T]

    Permalink
    Definition Classes
    SparkContext
  91. def range(start: Long, end: Long, step: Long, numSlices: Int): RDD[Long]

    Permalink
    Definition Classes
    SparkContext
  92. def register(acc: AccumulatorV2[_, _], name: String): Unit

    Permalink
    Definition Classes
    SparkContext
  93. def register(acc: AccumulatorV2[_, _]): Unit

    Permalink
    Definition Classes
    SparkContext
  94. def removeSparkListener(listener: SparkListenerInterface): Unit

    Permalink
    Definition Classes
    SparkContext
    Annotations
    @DeveloperApi()
  95. def requestExecutors(numAdditionalExecutors: Int): Boolean

    Permalink
    Definition Classes
    SparkContext
    Annotations
    @DeveloperApi()
  96. def requestTotalExecutors(numExecutors: Int, localityAwareTasks: Int, hostToLocalTaskCount: Map[String, Int]): Boolean

    Permalink
    Definition Classes
    SparkContext
    Annotations
    @DeveloperApi()
  97. def runApproximateJob[T, U, R](rdd: RDD[T], func: (TaskContext, Iterator[T]) ⇒ U, evaluator: ApproximateEvaluator[U, R], timeout: Long): PartialResult[R]

    Permalink
    Definition Classes
    SparkContext
    Annotations
    @DeveloperApi()
  98. def runJob[T, U](rdd: RDD[T], processPartition: (Iterator[T]) ⇒ U, resultHandler: (Int, U) ⇒ Unit)(implicit arg0: ClassTag[U]): Unit

    Permalink
    Definition Classes
    SparkContext
  99. def runJob[T, U](rdd: RDD[T], processPartition: (TaskContext, Iterator[T]) ⇒ U, resultHandler: (Int, U) ⇒ Unit)(implicit arg0: ClassTag[U]): Unit

    Permalink
    Definition Classes
    SparkContext
  100. def runJob[T, U](rdd: RDD[T], func: (Iterator[T]) ⇒ U)(implicit arg0: ClassTag[U]): Array[U]

    Permalink
    Definition Classes
    SparkContext
  101. def runJob[T, U](rdd: RDD[T], func: (TaskContext, Iterator[T]) ⇒ U)(implicit arg0: ClassTag[U]): Array[U]

    Permalink
    Definition Classes
    SparkContext
  102. def runJob[T, U](rdd: RDD[T], func: (Iterator[T]) ⇒ U, partitions: Seq[Int])(implicit arg0: ClassTag[U]): Array[U]

    Permalink
    Definition Classes
    SparkContext
  103. def runJob[T, U](rdd: RDD[T], func: (TaskContext, Iterator[T]) ⇒ U, partitions: Seq[Int])(implicit arg0: ClassTag[U]): Array[U]

    Permalink
    Definition Classes
    SparkContext
  104. def runJob[T, U](rdd: RDD[T], func: (TaskContext, Iterator[T]) ⇒ U, partitions: Seq[Int], resultHandler: (Int, U) ⇒ Unit)(implicit arg0: ClassTag[U]): Unit

    Permalink
    Definition Classes
    SparkContext
  105. def sequenceFile[K, V](path: String, minPartitions: Int)(implicit km: ClassTag[K], vm: ClassTag[V], kcf: () ⇒ WritableConverter[K], vcf: () ⇒ WritableConverter[V]): RDD[(K, V)]

    Permalink
    Definition Classes
    SparkContext
  106. def sequenceFile[K, V](path: String, keyClass: Class[K], valueClass: Class[V]): RDD[(K, V)]

    Permalink
    Definition Classes
    SparkContext
  107. def sequenceFile[K, V](path: String, keyClass: Class[K], valueClass: Class[V], minPartitions: Int): RDD[(K, V)]

    Permalink
    Definition Classes
    SparkContext
  108. def setCallSite(shortCallSite: String): Unit

    Permalink
    Definition Classes
    SparkContext
  109. def setCheckpointDir(directory: String): Unit

    Permalink
    Definition Classes
    SparkContext
  110. def setJobDescription(value: String): Unit

    Permalink
    Definition Classes
    SparkContext
  111. def setJobGroup(groupId: String, description: String, interruptOnCancel: Boolean): Unit

    Permalink
    Definition Classes
    SparkContext
  112. def setLocalProperty(key: String, value: String): Unit

    Permalink
    Definition Classes
    SparkContext
  113. def setLogLevel(logLevel: String): Unit

    Permalink
    Definition Classes
    SparkContext
  114. val sparkUser: String

    Permalink
    Definition Classes
    SparkContext
  115. val startTime: Long

    Permalink
    Definition Classes
    SparkContext
  116. def statusTracker: SparkStatusTracker

    Permalink
    Definition Classes
    SparkContext
  117. def stop(): Unit

    Permalink
    Definition Classes
    SparkContext
  118. def submitJob[T, U, R](rdd: RDD[T], processPartition: (Iterator[T]) ⇒ U, partitions: Seq[Int], resultHandler: (Int, U) ⇒ Unit, resultFunc: ⇒ R): SimpleFutureAction[R]

    Permalink
    Definition Classes
    SparkContext
  119. final def synchronized[T0](arg0: ⇒ T0): T0

    Permalink
    Definition Classes
    AnyRef
  120. def textFile(path: String, minPartitions: Int): RDD[String]

    Permalink
    Definition Classes
    SparkContext
  121. def toString(): String

    Permalink
    Definition Classes
    AnyRef → Any
  122. def uiWebUrl: Option[String]

    Permalink
    Definition Classes
    SparkContext
  123. def union[T](first: RDD[T], rest: RDD[T]*)(implicit arg0: ClassTag[T]): RDD[T]

    Permalink
    Definition Classes
    SparkContext
  124. def union[T](rdds: Seq[RDD[T]])(implicit arg0: ClassTag[T]): RDD[T]

    Permalink
    Definition Classes
    SparkContext
  125. def version: String

    Permalink
    Definition Classes
    SparkContext
  126. final def wait(): Unit

    Permalink
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  127. final def wait(arg0: Long, arg1: Int): Unit

    Permalink
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  128. final def wait(arg0: Long): Unit

    Permalink
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  129. def wholeTextFiles(path: String, minPartitions: Int): RDD[(String, String)]

    Permalink
    Definition Classes
    SparkContext

Deprecated Value Members

  1. def accumulable[R, T](initialValue: R, name: String)(implicit param: AccumulableParam[R, T]): Accumulable[R, T]

    Permalink
    Definition Classes
    SparkContext
    Annotations
    @deprecated
    Deprecated

    (Since version 2.0.0) use AccumulatorV2

  2. def accumulable[R, T](initialValue: R)(implicit param: AccumulableParam[R, T]): Accumulable[R, T]

    Permalink
    Definition Classes
    SparkContext
    Annotations
    @deprecated
    Deprecated

    (Since version 2.0.0) use AccumulatorV2

  3. def accumulableCollection[R, T](initialValue: R)(implicit arg0: (R) ⇒ Growable[T] with TraversableOnce[T] with Serializable, arg1: ClassTag[R]): Accumulable[R, T]

    Permalink
    Definition Classes
    SparkContext
    Annotations
    @deprecated
    Deprecated

    (Since version 2.0.0) use AccumulatorV2

  4. def accumulator[T](initialValue: T, name: String)(implicit param: AccumulatorParam[T]): Accumulator[T]

    Permalink
    Definition Classes
    SparkContext
    Annotations
    @deprecated
    Deprecated

    (Since version 2.0.0) use AccumulatorV2

  5. def accumulator[T](initialValue: T)(implicit param: AccumulatorParam[T]): Accumulator[T]

    Permalink
    Definition Classes
    SparkContext
    Annotations
    @deprecated
    Deprecated

    (Since version 2.0.0) use AccumulatorV2

  6. def getExecutorStorageStatus: Array[StorageStatus]

    Permalink
    Definition Classes
    SparkContext
    Annotations
    @DeveloperApi() @deprecated
    Deprecated

    (Since version 2.2.0) This method may change or be removed in a future release.

Inherited from SparkContext

Inherited from Logging

Inherited from AnyRef

Inherited from Any

Ungrouped