spark

PairRDDFunctions

class PairRDDFunctions[K, V] extends Logging with Serializable

Extra functions available on RDDs of (key, value) pairs through an implicit conversion.

Linear Supertypes
Serializable, Serializable, Logging, AnyRef, Any
Ordering
  1. Alphabetic
  2. By inheritance
Inherited
  1. Hide All
  2. Show all
  1. PairRDDFunctions
  2. Serializable
  3. Serializable
  4. Logging
  5. AnyRef
  6. Any
Visibility
  1. Public
  2. All

Instance Constructors

  1. new PairRDDFunctions(self: RDD[(K, V)])(implicit arg0: ClassManifest[K], arg1: ClassManifest[V])

Value Members

  1. final def !=(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef

  2. final def !=(arg0: Any): Boolean

    Definition Classes
    Any

  3. final def ##(): Int

    Definition Classes
    AnyRef → Any

  4. final def ==(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef

  5. final def ==(arg0: Any): Boolean

    Definition Classes
    Any

  6. final def asInstanceOf[T0]: T0

    Definition Classes
    Any

  7. def clone(): AnyRef

    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws()

  8. def cogroup[W1, W2](other1: RDD[(K, W1)], other2: RDD[(K, W2)], numSplits: Int): RDD[(K, (Seq[V], Seq[W1], Seq[W2]))]

  9. def cogroup[W](other: RDD[(K, W)], numSplits: Int): RDD[(K, (Seq[V], Seq[W]))]

  10. def cogroup[W1, W2](other1: RDD[(K, W1)], other2: RDD[(K, W2)]): RDD[(K, (Seq[V], Seq[W1], Seq[W2]))]

  11. def cogroup[W](other: RDD[(K, W)]): RDD[(K, (Seq[V], Seq[W]))]

  12. def cogroup[W1, W2](other1: RDD[(K, W1)], other2: RDD[(K, W2)], partitioner: Partitioner): RDD[(K, (Seq[V], Seq[W1], Seq[W2]))]

  13. def cogroup[W](other: RDD[(K, W)], partitioner: Partitioner): RDD[(K, (Seq[V], Seq[W]))]

  14. def collectAsMap(): Map[K, V]

  15. def combineByKey[C](createCombiner: (V) ⇒ C, mergeValue: (C, V) ⇒ C, mergeCombiners: (C, C) ⇒ C): RDD[(K, C)]

  16. def combineByKey[C](createCombiner: (V) ⇒ C, mergeValue: (C, V) ⇒ C, mergeCombiners: (C, C) ⇒ C, numSplits: Int): RDD[(K, C)]

  17. def combineByKey[C](createCombiner: (V) ⇒ C, mergeValue: (C, V) ⇒ C, mergeCombiners: (C, C) ⇒ C, partitioner: Partitioner): RDD[(K, C)]

  18. def defaultPartitioner(rdds: spark.RDD[_]*): Partitioner

    Choose a partitioner to use for a cogroup-like operation between a number of RDDs.

    Choose a partitioner to use for a cogroup-like operation between a number of RDDs. If any of the RDDs already has a partitioner, choose that one, otherwise use a default HashPartitioner.

  19. final def eq(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef

  20. def equals(arg0: Any): Boolean

    Definition Classes
    AnyRef → Any

  21. def finalize(): Unit

    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws()

  22. def flatMapValues[U](f: (V) ⇒ TraversableOnce[U]): RDD[(K, U)]

  23. final def getClass(): java.lang.Class[_]

    Definition Classes
    AnyRef → Any

  24. def getKeyClass(): java.lang.Class[_]

  25. def getValueClass(): java.lang.Class[_]

  26. def groupByKey(): RDD[(K, Seq[V])]

  27. def groupByKey(numSplits: Int): RDD[(K, Seq[V])]

  28. def groupByKey(partitioner: Partitioner): RDD[(K, Seq[V])]

  29. def groupWith[W1, W2](other1: RDD[(K, W1)], other2: RDD[(K, W2)]): RDD[(K, (Seq[V], Seq[W1], Seq[W2]))]

  30. def groupWith[W](other: RDD[(K, W)]): RDD[(K, (Seq[V], Seq[W]))]

  31. def hashCode(): Int

    Definition Classes
    AnyRef → Any

  32. def initLogging(): Unit

    Definition Classes
    Logging

  33. final def isInstanceOf[T0]: Boolean

    Definition Classes
    Any

  34. def join[W](other: RDD[(K, W)], numSplits: Int): RDD[(K, (V, W))]

  35. def join[W](other: RDD[(K, W)]): RDD[(K, (V, W))]

  36. def join[W](other: RDD[(K, W)], partitioner: Partitioner): RDD[(K, (V, W))]

  37. def leftOuterJoin[W](other: RDD[(K, W)], numSplits: Int): RDD[(K, (V, Option[W]))]

  38. def leftOuterJoin[W](other: RDD[(K, W)]): RDD[(K, (V, Option[W]))]

  39. def leftOuterJoin[W](other: RDD[(K, W)], partitioner: Partitioner): RDD[(K, (V, Option[W]))]

  40. def log: Logger

    Definition Classes
    Logging

  41. def logDebug(msg: ⇒ String, throwable: Throwable): Unit

    Definition Classes
    Logging

  42. def logDebug(msg: ⇒ String): Unit

    Definition Classes
    Logging

  43. def logError(msg: ⇒ String, throwable: Throwable): Unit

    Definition Classes
    Logging

  44. def logError(msg: ⇒ String): Unit

    Definition Classes
    Logging

  45. def logInfo(msg: ⇒ String, throwable: Throwable): Unit

    Definition Classes
    Logging

  46. def logInfo(msg: ⇒ String): Unit

    Definition Classes
    Logging

  47. def logWarning(msg: ⇒ String, throwable: Throwable): Unit

    Definition Classes
    Logging

  48. def logWarning(msg: ⇒ String): Unit

    Definition Classes
    Logging

  49. def lookup(key: K): Seq[V]

  50. def mapValues[U](f: (V) ⇒ U): RDD[(K, U)]

  51. final def ne(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef

  52. final def notify(): Unit

    Definition Classes
    AnyRef

  53. final def notifyAll(): Unit

    Definition Classes
    AnyRef

  54. def partitionBy(partitioner: Partitioner): RDD[(K, V)]

  55. def reduceByKey(func: (V, V) ⇒ V): RDD[(K, V)]

  56. def reduceByKey(func: (V, V) ⇒ V, numSplits: Int): RDD[(K, V)]

  57. def reduceByKey(partitioner: Partitioner, func: (V, V) ⇒ V): RDD[(K, V)]

  58. def reduceByKeyToDriver(func: (V, V) ⇒ V): Map[K, V]

  59. def rightOuterJoin[W](other: RDD[(K, W)], numSplits: Int): RDD[(K, (Option[V], W))]

  60. def rightOuterJoin[W](other: RDD[(K, W)]): RDD[(K, (Option[V], W))]

  61. def rightOuterJoin[W](other: RDD[(K, W)], partitioner: Partitioner): RDD[(K, (Option[V], W))]

  62. def saveAsHadoopDataset(conf: JobConf): Unit

  63. def saveAsHadoopFile(path: String, keyClass: Class[_], valueClass: Class[_], outputFormatClass: Class[_ <: org.apache.hadoop.mapred.OutputFormat[_, _]], conf: JobConf = new JobConf): Unit

  64. def saveAsHadoopFile[F <: OutputFormat[K, V]](path: String)(implicit fm: ClassManifest[F]): Unit

  65. def saveAsNewAPIHadoopFile(path: String, keyClass: Class[_], valueClass: Class[_], outputFormatClass: Class[_ <: org.apache.hadoop.mapreduce.OutputFormat[_, _]], conf: Configuration): Unit

  66. def saveAsNewAPIHadoopFile(path: String, keyClass: Class[_], valueClass: Class[_], outputFormatClass: Class[_ <: org.apache.hadoop.mapreduce.OutputFormat[_, _]]): Unit

  67. def saveAsNewAPIHadoopFile[F <: OutputFormat[K, V]](path: String)(implicit fm: ClassManifest[F]): Unit

  68. final def synchronized[T0](arg0: ⇒ T0): T0

    Definition Classes
    AnyRef

  69. def toString(): String

    Definition Classes
    AnyRef → Any

  70. final def wait(): Unit

    Definition Classes
    AnyRef
    Annotations
    @throws()

  71. final def wait(arg0: Long, arg1: Int): Unit

    Definition Classes
    AnyRef
    Annotations
    @throws()

  72. final def wait(arg0: Long): Unit

    Definition Classes
    AnyRef
    Annotations
    @throws()

Inherited from Serializable

Inherited from Serializable

Inherited from Logging

Inherited from AnyRef

Inherited from Any