spark.ShuffledRDD

Instance Constructors

new ShuffledRDD(parent: RDD[(K, V)], aggregator: Aggregator[K, V, C], part: Partitioner)

Value Members

final def !=(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def !=(arg0: Any): Boolean

Definition Classes
Any
final def ##(): Int

Definition Classes
AnyRef → Any
def ++(other: RDD[(K, C)]): RDD[(K, C)]

Definition Classes
RDD
final def ==(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def ==(arg0: Any): Boolean

Definition Classes
Any
def aggregate[U](zeroValue: U)(seqOp: (U, (K, C)) ⇒ U, combOp: (U, U) ⇒ U)(implicit arg0: ClassManifest[U]): U

Aggregate the elements of each partition, and then the results for all the partitions, using given combine functions and a neutral "zero value".
Aggregate the elements of each partition, and then the results for all the partitions, using given combine functions and a neutral "zero value". This function can return a different result type, U, than the type of this RDD, T. Thus, we need one operation for merging a T into an U and one operation for merging two U's, as in scala.TraversableOnce. Both of these functions are allowed to modify and return their first argument instead of creating a new U to avoid memory allocation.

Definition Classes
RDD
final def asInstanceOf[T0]: T0

Definition Classes
Any
def cache(): RDD[(K, C)]

Definition Classes
RDD
def cartesian[U](other: RDD[U])(implicit arg0: ClassManifest[U]): RDD[((K, C), U)]

Definition Classes
RDD
def clone(): AnyRef

Attributes
protected[lang]
Definition Classes
AnyRef
Annotations
@throws()
def collect(): Array[(K, C)]

Definition Classes
RDD
def compute(split: Split): Iterator[(K, C)]

Definition Classes
ShuffledRDD → RDD
def context: SparkContext

Definition Classes
RDD
def count(): Long

Definition Classes
RDD
val dep: ShuffleDependency[K, V, C]
val dependencies: List[ShuffleDependency[K, V, C]]

Definition Classes
ShuffledRDD → RDD
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def filter(f: ((K, C)) ⇒ Boolean): RDD[(K, C)]

Definition Classes
RDD
def finalize(): Unit

Attributes
protected[lang]
Definition Classes
AnyRef
Annotations
@throws()
def first(): (K, C)

Definition Classes
RDD
def flatMap[U](f: ((K, C)) ⇒ TraversableOnce[U])(implicit arg0: ClassManifest[U]): RDD[U]

Definition Classes
RDD
def fold(zeroValue: (K, C))(op: ((K, C), (K, C)) ⇒ (K, C)): (K, C)

Aggregate the elements of each partition, and then the results for all the partitions, using a given associative function and a neutral "zero value".
Aggregate the elements of each partition, and then the results for all the partitions, using a given associative function and a neutral "zero value". The function op(t1, t2) is allowed to modify t1 and return it as its result value to avoid object allocation; however, it should not modify t2.

Definition Classes
RDD
def foreach(f: ((K, C)) ⇒ Unit): Unit

Definition Classes
RDD
final def getClass(): java.lang.Class[_]

Definition Classes
AnyRef → Any
def glom(): RDD[Array[(K, C)]]

Definition Classes
RDD
def groupBy[K](f: ((K, C)) ⇒ K)(implicit arg0: ClassManifest[K]): RDD[(K, Seq[(K, C)])]

Definition Classes
RDD
def groupBy[K](f: ((K, C)) ⇒ K, numSplits: Int)(implicit arg0: ClassManifest[K]): RDD[(K, Seq[(K, C)])]

Definition Classes
RDD
def hashCode(): Int

Definition Classes
AnyRef → Any
val id: Int

Definition Classes
RDD
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
final def iterator(split: Split): Iterator[(K, C)]

Definition Classes
RDD
def map[U](f: ((K, C)) ⇒ U)(implicit arg0: ClassManifest[U]): RDD[U]

Definition Classes
RDD
def mapPartitions[U](f: (Iterator[(K, C)]) ⇒ Iterator[U])(implicit arg0: ClassManifest[U]): RDD[U]

Definition Classes
RDD
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
val partitioner: Some[Partitioner]

Definition Classes
ShuffledRDD → RDD
def pipe(command: Seq[String], env: Map[String, String]): RDD[String]

Definition Classes
RDD
def pipe(command: Seq[String]): RDD[String]

Definition Classes
RDD
def pipe(command: String): RDD[String]

Definition Classes
RDD
def preferredLocations(split: Split): scala.collection.immutable.Nil.type

Definition Classes
ShuffledRDD → RDD
def reduce(f: ((K, C), (K, C)) ⇒ (K, C)): (K, C)

Definition Classes
RDD
def sample(withReplacement: Boolean, fraction: Double, seed: Int): RDD[(K, C)]

Definition Classes
RDD
def saveAsObjectFile(path: String): Unit

Definition Classes
RDD
def saveAsTextFile(path: String): Unit

Definition Classes
RDD
def splits: Array[Split]

Definition Classes
ShuffledRDD → RDD
val splits_: Array[Split]
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def take(num: Int): Array[(K, C)]

Take the first num elements of the RDD.
Take the first num elements of the RDD. This currently scans the partitions *one by one*, so it will be slow if a lot of partitions are required. In that case, use collect() to get the whole RDD instead.

Definition Classes
RDD
def takeSample(withReplacement: Boolean, num: Int, seed: Int): Array[(K, C)]

Definition Classes
RDD
def toArray(): Array[(K, C)]

Definition Classes
RDD
def toString(): String

Definition Classes
AnyRef → Any
def union(other: RDD[(K, C)]): RDD[(K, C)]

Definition Classes
RDD
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws()
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws()
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws()

ShuffledRDD

class ShuffledRDD[K, V, C] extends RDD[(K, C)]

Instance Constructors

new ShuffledRDD(parent: RDD[(K, V)], aggregator: Aggregator[K, V, C], part: Partitioner)

Value Members

final def !=(arg0: AnyRef): Boolean

final def !=(arg0: Any): Boolean

final def ##(): Int

def ++(other: RDD[(K, C)]): RDD[(K, C)]

final def ==(arg0: AnyRef): Boolean

final def ==(arg0: Any): Boolean

def aggregate[U](zeroValue: U)(seqOp: (U, (K, C)) ⇒ U, combOp: (U, U) ⇒ U)(implicit arg0: ClassManifest[U]): U

final def asInstanceOf[T0]: T0

def cache(): RDD[(K, C)]

def cartesian[U](other: RDD[U])(implicit arg0: ClassManifest[U]): RDD[((K, C), U)]

def clone(): AnyRef

def collect(): Array[(K, C)]

def compute(split: Split): Iterator[(K, C)]

def context: SparkContext

def count(): Long

val dep: ShuffleDependency[K, V, C]

val dependencies: List[ShuffleDependency[K, V, C]]

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def filter(f: ((K, C)) ⇒ Boolean): RDD[(K, C)]

def finalize(): Unit

def first(): (K, C)

def flatMap[U](f: ((K, C)) ⇒ TraversableOnce[U])(implicit arg0: ClassManifest[U]): RDD[U]

def fold(zeroValue: (K, C))(op: ((K, C), (K, C)) ⇒ (K, C)): (K, C)

def foreach(f: ((K, C)) ⇒ Unit): Unit

final def getClass(): java.lang.Class[_]

def glom(): RDD[Array[(K, C)]]

def groupBy[K](f: ((K, C)) ⇒ K)(implicit arg0: ClassManifest[K]): RDD[(K, Seq[(K, C)])]

def groupBy[K](f: ((K, C)) ⇒ K, numSplits: Int)(implicit arg0: ClassManifest[K]): RDD[(K, Seq[(K, C)])]

def hashCode(): Int

val id: Int

final def isInstanceOf[T0]: Boolean

final def iterator(split: Split): Iterator[(K, C)]

def map[U](f: ((K, C)) ⇒ U)(implicit arg0: ClassManifest[U]): RDD[U]

def mapPartitions[U](f: (Iterator[(K, C)]) ⇒ Iterator[U])(implicit arg0: ClassManifest[U]): RDD[U]

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

val partitioner: Some[Partitioner]

def pipe(command: Seq[String], env: Map[String, String]): RDD[String]

def pipe(command: Seq[String]): RDD[String]

def pipe(command: String): RDD[String]

def preferredLocations(split: Split): scala.collection.immutable.Nil.type

def reduce(f: ((K, C), (K, C)) ⇒ (K, C)): (K, C)

def sample(withReplacement: Boolean, fraction: Double, seed: Int): RDD[(K, C)]

def saveAsObjectFile(path: String): Unit

def saveAsTextFile(path: String): Unit

def splits: Array[Split]

val splits_: Array[Split]

final def synchronized[T0](arg0: ⇒ T0): T0

def take(num: Int): Array[(K, C)]

def takeSample(withReplacement: Boolean, num: Int, seed: Int): Array[(K, C)]

def toArray(): Array[(K, C)]

def toString(): String

def union(other: RDD[(K, C)]): RDD[(K, C)]

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from RDD[(K, C)]

Inherited from Serializable

Inherited from Serializable

Inherited from AnyRef

Inherited from Any