ShuffleRegionJoin

A trait describing join implementations that are based on a sort-merge join.

T: The type of the left RDD.
U: The type of the right RDD.
RT: The type of data yielded by the left RDD at the output of the join. This may not match T if the join is an outer join, etc.
RU: The type of data yielded by the right RDD at the output of the join.

Linear Supertypes

RegionJoin[T, U, RT, RU], Serializable, Serializable, AnyRef, Any

Known Subclasses

FullOuterShuffleRegionJoin, InnerShuffleRegionJoin, InnerShuffleRegionJoinAndGroupByLeft, LeftOuterShuffleRegionJoin, LeftOuterShuffleRegionJoinAndGroupByLeft, RightOuterShuffleRegionJoinAndGroupByLeft, SortedIntervalPartitionJoinWithVictims, VictimlessSortedIntervalPartitionJoin

Abstract Value Members

abstract def advanceCache(cache: SetTheoryCache[U, RT, RU], right: BufferedIterator[(ReferenceRegion, U)], until: ReferenceRegion): Unit

Attributes
protected
abstract def emptyFn(left: Iterator[(ReferenceRegion, T)], right: Iterator[(ReferenceRegion, U)]): Iterator[(RT, RU)]

Attributes
protected
abstract def finalizeHits(cache: SetTheoryCache[U, RT, RU], right: BufferedIterator[(ReferenceRegion, U)]): Iterable[(RT, RU)]

Attributes
protected
abstract val leftRdd: RDD[(ReferenceRegion, T)]

Attributes
protected
abstract def postProcessHits(iter: Iterable[U], currentLeft: T): Iterable[(RT, RU)]

Attributes
protected
abstract def pruneCache(cache: SetTheoryCache[U, RT, RU], to: ReferenceRegion): Unit

Attributes
protected
abstract val rightRdd: RDD[(ReferenceRegion, U)]

Attributes
protected

Concrete Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def compute(): RDD[(RT, RU)]

Performs a region join between two RDDs (shuffle join).
Performs a region join between two RDDs (shuffle join). All data should be pre-shuffled and copartitioned.
returns
An RDD of joins (x, y), where x is from leftRDD, y is from rightRDD, and the region corresponding to x overlaps the region corresponding to y.
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def hashCode(): Int

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def makeIterator(leftIter: Iterator[(ReferenceRegion, T)], rightIter: Iterator[(ReferenceRegion, U)]): Iterator[(RT, RU)]

Attributes
protected
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def partitionAndJoin(left: RDD[(ReferenceRegion, T)], right: RDD[(ReferenceRegion, U)]): RDD[(RT, RU)]

Performs a region join between two RDDs.
Performs a region join between two RDDs.
returns
An RDD of pairs (x, y), where x is from baseRDD, y is from joinedRDD, and the region corresponding to x overlaps the region corresponding to y.

Definition Classes
ShuffleRegionJoin → RegionJoin
def processHits(cache: SetTheoryCache[U, RT, RU], currentLeft: T, currentLeftRegion: ReferenceRegion): Iterable[(RT, RU)]

Process hits for a given object in left.
Process hits for a given object in left.
cache
The cache containing potential hits.
currentLeft
The current object from the left
currentLeftRegion
The ReferenceRegion of currentLeft.
returns
An iterator containing all hits, formatted by postProcessHits.

Attributes
protected
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Doc: package rdd

sealed abstract class ShuffleRegionJoin[T, U, RT, RU] extends RegionJoin[T, U, RT, RU]

Abstract Value Members

abstract def advanceCache(cache: SetTheoryCache[U, RT, RU], right: BufferedIterator[(ReferenceRegion, U)], until: ReferenceRegion): Unit

abstract def emptyFn(left: Iterator[(ReferenceRegion, T)], right: Iterator[(ReferenceRegion, U)]): Iterator[(RT, RU)]

abstract def finalizeHits(cache: SetTheoryCache[U, RT, RU], right: BufferedIterator[(ReferenceRegion, U)]): Iterable[(RT, RU)]

abstract val leftRdd: RDD[(ReferenceRegion, T)]

abstract def postProcessHits(iter: Iterable[U], currentLeft: T): Iterable[(RT, RU)]

abstract def pruneCache(cache: SetTheoryCache[U, RT, RU], to: ReferenceRegion): Unit

abstract val rightRdd: RDD[(ReferenceRegion, U)]

Concrete Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def clone(): AnyRef

def compute(): RDD[(RT, RU)]

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def getClass(): Class[_]

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

def makeIterator(leftIter: Iterator[(ReferenceRegion, T)], rightIter: Iterator[(ReferenceRegion, U)]): Iterator[(RT, RU)]

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def partitionAndJoin(left: RDD[(ReferenceRegion, T)], right: RDD[(ReferenceRegion, U)]): RDD[(RT, RU)]

def processHits(cache: SetTheoryCache[U, RT, RU], currentLeft: T, currentLeftRegion: ReferenceRegion): Iterable[(RT, RU)]

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from RegionJoin[T, U, RT, RU]

Inherited from Serializable

Inherited from Serializable

Inherited from AnyRef

Inherited from Any

Ungrouped