SparkContext

Instance Constructors

new SparkContext(sparkConf: SparkConf)

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def accumulable[R, T](initialValue: R, name: String)(implicit param: AccumulableParam[R, T]): Accumulable[R, T]

Definition Classes
SparkContext
def accumulable[R, T](initialValue: R)(implicit param: AccumulableParam[R, T]): Accumulable[R, T]

Definition Classes
SparkContext
def accumulableCollection[R, T](initialValue: R)(implicit arg0: (R) ⇒ Growable[T] with TraversableOnce[T] with Serializable, arg1: ClassTag[R]): Accumulable[R, T]

Definition Classes
SparkContext
def accumulator[T](initialValue: T, name: String)(implicit param: AccumulatorParam[T]): Accumulator[T]

Definition Classes
SparkContext
def accumulator[T](initialValue: T)(implicit param: AccumulatorParam[T]): Accumulator[T]

Definition Classes
SparkContext
def addFile(path: String): Unit

Definition Classes
SparkContext
def addJar(path: String): Unit

Definition Classes
SparkContext
def addSparkListener(listener: SparkListener): Unit

Definition Classes
SparkContext
Annotations
@DeveloperApi()
val appName: String

Definition Classes
SparkContext
val applicationId: String

Definition Classes
SparkContext
final def asInstanceOf[T0]: T0

Definition Classes
Any
def binaryFiles(path: String, minPartitions: Int): RDD[(String, PortableDataStream)]

Definition Classes
SparkContext
Annotations
@Experimental()
def binaryRecords(path: String, recordLength: Int, conf: Configuration): RDD[Array[Byte]]

Definition Classes
SparkContext
Annotations
@Experimental()
def broadcast[T](value: T)(implicit arg0: ClassTag[T]): Broadcast[T]

Definition Classes
SparkContext
def cancelAllJobs(): Unit

Definition Classes
SparkContext
def cancelJobGroup(groupId: String): Unit

Definition Classes
SparkContext
def checkpointFile[T](path: String)(implicit arg0: ClassTag[T]): RDD[T]

Attributes
protected[org.apache.spark]
Definition Classes
SparkContext
def clearCallSite(): Unit

Definition Classes
SparkContext
def clearJobGroup(): Unit

Definition Classes
SparkContext
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def defaultMinPartitions: Int

Definition Classes
SparkContext
def defaultParallelism: Int

Definition Classes
SparkContext
def emptyRDD[T](implicit arg0: ClassTag[T]): EmptyRDD[T]

Definition Classes
SparkContext
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
val files: Seq[String]

Definition Classes
SparkContext
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
def getAllPools: Seq[Schedulable]

Definition Classes
SparkContext
Annotations
@DeveloperApi()
def getCheckpointDir: Option[String]

Definition Classes
SparkContext
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getConf: SparkConf

Definition Classes
SparkContext
def getExecutorMemoryStatus: Map[String, (Long, Long)]

Definition Classes
SparkContext
def getExecutorStorageStatus: Array[StorageStatus]

Definition Classes
SparkContext
Annotations
@DeveloperApi()
def getLocalProperty(key: String): String

Definition Classes
SparkContext
def getPersistentRDDs: Map[Int, RDD[_]]

Definition Classes
SparkContext
def getPoolForName(pool: String): Option[Schedulable]

Definition Classes
SparkContext
Annotations
@DeveloperApi()
def getRDDStorageInfo: Array[RDDInfo]

Definition Classes
SparkContext
Annotations
@DeveloperApi()
def getSchedulingMode: SchedulingMode

Definition Classes
SparkContext
val hadoopConfiguration: Configuration

Definition Classes
SparkContext
def hadoopFile[K, V, F <: InputFormat[K, V]](path: String)(implicit km: ClassTag[K], vm: ClassTag[V], fm: ClassTag[F]): RDD[(K, V)]

Definition Classes
SparkContext
def hadoopFile[K, V, F <: InputFormat[K, V]](path: String, minPartitions: Int)(implicit km: ClassTag[K], vm: ClassTag[V], fm: ClassTag[F]): RDD[(K, V)]

Definition Classes
SparkContext
def hadoopFile[K, V](path: String, inputFormatClass: Class[_ <: InputFormat[K, V]], keyClass: Class[K], valueClass: Class[V], minPartitions: Int): RDD[(K, V)]

Definition Classes
SparkContext
def hadoopRDD[K, V](conf: JobConf, inputFormatClass: Class[_ <: InputFormat[K, V]], keyClass: Class[K], valueClass: Class[V], minPartitions: Int): RDD[(K, V)]

Definition Classes
SparkContext
def hashCode(): Int

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
val isLocal: Boolean

Definition Classes
SparkContext
def isTraceEnabled(): Boolean

Attributes
protected
Definition Classes
Logging
val jars: Seq[String]

Definition Classes
SparkContext
def killExecutor(executorId: String): Boolean

Definition Classes
SparkContext → ExecutorAllocationClient
Annotations
@DeveloperApi()
def killExecutors(executorIds: Seq[String]): Boolean

Definition Classes
SparkContext → ExecutorAllocationClient
Annotations
@DeveloperApi()
def log: Logger

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logName: String

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def makeRDD[T](seq: Seq[(T, Seq[String])])(implicit arg0: ClassTag[T]): RDD[T]

Definition Classes
SparkContext
def makeRDD[T](seq: Seq[T], numSlices: Int)(implicit arg0: ClassTag[T]): RDD[T]

Definition Classes
SparkContext
val master: String

Definition Classes
SparkContext
val metricsSystem: MetricsSystem

Definition Classes
SparkContext
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def newAPIHadoopFile[K, V, F <: InputFormat[K, V]](path: String, fClass: Class[F], kClass: Class[K], vClass: Class[V], conf: Configuration): RDD[(K, V)]

Definition Classes
SparkContext
def newAPIHadoopFile[K, V, F <: InputFormat[K, V]](path: String)(implicit km: ClassTag[K], vm: ClassTag[V], fm: ClassTag[F]): RDD[(K, V)]

Definition Classes
SparkContext
def newAPIHadoopRDD[K, V, F <: InputFormat[K, V]](conf: Configuration, fClass: Class[F], kClass: Class[K], vClass: Class[V]): RDD[(K, V)]

Definition Classes
SparkContext
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def objectFile[T](path: String, minPartitions: Int)(implicit arg0: ClassTag[T]): RDD[T]

Definition Classes
SparkContext
def parallelize[T](seq: Seq[T], numSlices: Int)(implicit arg0: ClassTag[T]): RDD[T]

Definition Classes
SparkContext
def requestExecutors(numAdditionalExecutors: Int): Boolean

Definition Classes
SparkContext → ExecutorAllocationClient
Annotations
@DeveloperApi()
def runApproximateJob[T, U, R](rdd: RDD[T], func: (TaskContext, Iterator[T]) ⇒ U, evaluator: ApproximateEvaluator[U, R], timeout: Long): PartialResult[R]

Definition Classes
SparkContext
Annotations
@DeveloperApi()
def runJob[T, U](rdd: RDD[T], processPartition: (Iterator[T]) ⇒ U, resultHandler: (Int, U) ⇒ Unit)(implicit arg0: ClassTag[U]): Unit

Definition Classes
SparkContext
def runJob[T, U](rdd: RDD[T], processPartition: (TaskContext, Iterator[T]) ⇒ U, resultHandler: (Int, U) ⇒ Unit)(implicit arg0: ClassTag[U]): Unit

Definition Classes
SparkContext
def runJob[T, U](rdd: RDD[T], func: (Iterator[T]) ⇒ U)(implicit arg0: ClassTag[U]): Array[U]

Definition Classes
SparkContext
def runJob[T, U](rdd: RDD[T], func: (TaskContext, Iterator[T]) ⇒ U)(implicit arg0: ClassTag[U]): Array[U]

Definition Classes
SparkContext
def runJob[T, U](rdd: RDD[T], func: (Iterator[T]) ⇒ U, partitions: Seq[Int], allowLocal: Boolean)(implicit arg0: ClassTag[U]): Array[U]

Definition Classes
SparkContext
def runJob[T, U](rdd: RDD[T], func: (TaskContext, Iterator[T]) ⇒ U, partitions: Seq[Int], allowLocal: Boolean)(implicit arg0: ClassTag[U]): Array[U]

Definition Classes
SparkContext
def runJob[T, U](rdd: RDD[T], func: (TaskContext, Iterator[T]) ⇒ U, partitions: Seq[Int], allowLocal: Boolean, resultHandler: (Int, U) ⇒ Unit)(implicit arg0: ClassTag[U]): Unit

Definition Classes
SparkContext
def sequenceFile[K, V](path: String, minPartitions: Int)(implicit km: ClassTag[K], vm: ClassTag[V], kcf: () ⇒ WritableConverter[K], vcf: () ⇒ WritableConverter[V]): RDD[(K, V)]

Definition Classes
SparkContext
def sequenceFile[K, V](path: String, keyClass: Class[K], valueClass: Class[V]): RDD[(K, V)]

Definition Classes
SparkContext
def sequenceFile[K, V](path: String, keyClass: Class[K], valueClass: Class[V], minPartitions: Int): RDD[(K, V)]

Definition Classes
SparkContext
def setCallSite(shortCallSite: String): Unit

Definition Classes
SparkContext
def setCheckpointDir(directory: String): Unit

Definition Classes
SparkContext
def setJobGroup(groupId: String, description: String, interruptOnCancel: Boolean): Unit

Definition Classes
SparkContext
def setLocalProperty(key: String, value: String): Unit

Definition Classes
SparkContext
val sparkUser: String

Definition Classes
SparkContext
val startTime: Long

Definition Classes
SparkContext
val statusTracker: SparkStatusTracker

Definition Classes
SparkContext
def stop(): Unit

Definition Classes
SparkContext
def submitJob[T, U, R](rdd: RDD[T], processPartition: (Iterator[T]) ⇒ U, partitions: Seq[Int], resultHandler: (Int, U) ⇒ Unit, resultFunc: ⇒ R): SimpleFutureAction[R]

Definition Classes
SparkContext
Annotations
@Experimental()
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
val tachyonFolderName: String

Definition Classes
SparkContext
def textFile(path: String, minPartitions: Int): RDD[String]

Definition Classes
SparkContext
def toString(): String

Definition Classes
SparkContext → AnyRef → Any
def union[T](first: RDD[T], rest: RDD[T]*)(implicit arg0: ClassTag[T]): RDD[T]

Definition Classes
SparkContext
def union[T](rdds: Seq[RDD[T]])(implicit arg0: ClassTag[T]): RDD[T]

Definition Classes
SparkContext
def version: String

Definition Classes
SparkContext
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
def wholeTextFiles(path: String, minPartitions: Int): RDD[(String, String)]

Definition Classes
SparkContext

Deprecated Value Members

def clearFiles(): Unit

Definition Classes
SparkContext
Annotations
@deprecated
Deprecated
(Since version 1.0.0) adding files no longer creates local copies that need to be deleted
def clearJars(): Unit

Definition Classes
SparkContext
Annotations
@deprecated
Deprecated
(Since version 1.0.0) adding jars no longer creates local copies that need to be deleted
def defaultMinSplits: Int

Definition Classes
SparkContext
Annotations
@deprecated
Deprecated
(Since version 1.0.0) use defaultMinPartitions
def initLocalProperties(): Unit

Definition Classes
SparkContext
Annotations
@deprecated
Deprecated
(Since version 1.0.0) Properties no longer need to be explicitly initialized.
def setJobDescription(value: String): Unit

Definition Classes
SparkContext
Annotations
@deprecated
Deprecated
(Since version 0.8.1) use setJobGroup

Related Doc: package Spark

class SparkContext extends org.apache.spark.SparkContext

Instance Constructors

new SparkContext(sparkConf: SparkConf)

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def accumulable[R, T](initialValue: R, name: String)(implicit param: AccumulableParam[R, T]): Accumulable[R, T]

def accumulable[R, T](initialValue: R)(implicit param: AccumulableParam[R, T]): Accumulable[R, T]

def accumulableCollection[R, T](initialValue: R)(implicit arg0: (R) ⇒ Growable[T] with TraversableOnce[T] with Serializable, arg1: ClassTag[R]): Accumulable[R, T]

def accumulator[T](initialValue: T, name: String)(implicit param: AccumulatorParam[T]): Accumulator[T]

def accumulator[T](initialValue: T)(implicit param: AccumulatorParam[T]): Accumulator[T]

def addFile(path: String): Unit

def addJar(path: String): Unit

def addSparkListener(listener: SparkListener): Unit

val appName: String

val applicationId: String

final def asInstanceOf[T0]: T0

def binaryFiles(path: String, minPartitions: Int): RDD[(String, PortableDataStream)]

def binaryRecords(path: String, recordLength: Int, conf: Configuration): RDD[Array[Byte]]

def broadcast[T](value: T)(implicit arg0: ClassTag[T]): Broadcast[T]

def cancelAllJobs(): Unit

def cancelJobGroup(groupId: String): Unit

def checkpointFile[T](path: String)(implicit arg0: ClassTag[T]): RDD[T]

def clearCallSite(): Unit

def clearJobGroup(): Unit

def clone(): AnyRef

def defaultMinPartitions: Int

def defaultParallelism: Int

def emptyRDD[T](implicit arg0: ClassTag[T]): EmptyRDD[T]

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

val files: Seq[String]

def finalize(): Unit

def getAllPools: Seq[Schedulable]

def getCheckpointDir: Option[String]

final def getClass(): Class[_]

def getConf: SparkConf

def getExecutorMemoryStatus: Map[String, (Long, Long)]

def getExecutorStorageStatus: Array[StorageStatus]

def getLocalProperty(key: String): String

def getPersistentRDDs: Map[Int, RDD[_]]

def getPoolForName(pool: String): Option[Schedulable]

def getRDDStorageInfo: Array[RDDInfo]

def getSchedulingMode: SchedulingMode

val hadoopConfiguration: Configuration

def hadoopFile[K, V, F <: InputFormat[K, V]](path: String)(implicit km: ClassTag[K], vm: ClassTag[V], fm: ClassTag[F]): RDD[(K, V)]

def hadoopFile[K, V, F <: InputFormat[K, V]](path: String, minPartitions: Int)(implicit km: ClassTag[K], vm: ClassTag[V], fm: ClassTag[F]): RDD[(K, V)]

def hadoopFile[K, V](path: String, inputFormatClass: Class[_ <: InputFormat[K, V]], keyClass: Class[K], valueClass: Class[V], minPartitions: Int): RDD[(K, V)]

def hadoopRDD[K, V](conf: JobConf, inputFormatClass: Class[_ <: InputFormat[K, V]], keyClass: Class[K], valueClass: Class[V], minPartitions: Int): RDD[(K, V)]

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

val isLocal: Boolean

def isTraceEnabled(): Boolean

val jars: Seq[String]

def killExecutor(executorId: String): Boolean

def killExecutors(executorIds: Seq[String]): Boolean

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logName: String

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

def makeRDD[T](seq: Seq[(T, Seq[String])])(implicit arg0: ClassTag[T]): RDD[T]

def makeRDD[T](seq: Seq[T], numSlices: Int)(implicit arg0: ClassTag[T]): RDD[T]

val master: String

val metricsSystem: MetricsSystem

final def ne(arg0: AnyRef): Boolean

def newAPIHadoopFile[K, V, F <: InputFormat[K, V]](path: String, fClass: Class[F], kClass: Class[K], vClass: Class[V], conf: Configuration): RDD[(K, V)]

def newAPIHadoopFile[K, V, F <: InputFormat[K, V]](path: String)(implicit km: ClassTag[K], vm: ClassTag[V], fm: ClassTag[F]): RDD[(K, V)]

def newAPIHadoopRDD[K, V, F <: InputFormat[K, V]](conf: Configuration, fClass: Class[F], kClass: Class[K], vClass: Class[V]): RDD[(K, V)]

final def notify(): Unit