MongoPaginateByCountPartitioner

The pagination by count partitioner.

Paginates the collection into a maximum number of partitions.

Configuration Properties

The prefix when using sparkConf is: spark.mongodb.input.partitionerOptions followed by the property name:

partitionKey, the field to partition the collection by. The field should be indexed and contain unique values. Defaults to _id.
numberOfPartitions, the maximum number of partitions to create. Defaults to 64.

Note: This can be a expensive operation as it creates 1 cursor for every partition.

Since: 1.0

Linear Supertypes

MongoPaginationPartitioner, MongoPartitioner, Serializable, Serializable, Logging, LoggingTrait, AnyRef, Any

Known Subclasses

MongoPaginateByCountPartitioner

Instance Constructors

new MongoPaginateByCountPartitioner()

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
def calculatePartitions(connector: MongoConnector, readConfig: ReadConfig, partitionKey: String, count: Long, numDocumentsPerPartition: Int, matchQuery: Bson): Seq[BsonValue]

Attributes
protected
Definition Classes
MongoPaginationPartitioner
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def hashCode(): Int

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def log: Logger

Attributes
protected
Definition Classes
LoggingTrait
def logDebug(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
LoggingTrait
def logDebug(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
LoggingTrait
def logError(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
LoggingTrait
def logError(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
LoggingTrait
def logInfo(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
LoggingTrait
def logInfo(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
LoggingTrait
def logName: String

Attributes
protected
Definition Classes
LoggingTrait
def logTrace(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
LoggingTrait
def logTrace(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
LoggingTrait
def logWarning(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
LoggingTrait
def logWarning(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
LoggingTrait
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
val numberOfPartitionsProperty: String

The number of partitions property
val partitionKeyProperty: String

The partition key property
def partitions(connector: MongoConnector, readConfig: ReadConfig, pipeline: Array[BsonDocument]): Array[MongoPartition]

Calculate the Partitions
Calculate the Partitions
connector
the MongoConnector
readConfig
the com.mongodb.spark.config.ReadConfig
pipeline
the pipeline to apply if any. Note this pipeline may have been appended to during optimization.
returns
the partitions

Definition Classes
MongoPaginateByCountPartitioner → MongoPartitioner
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Docs: object MongoPaginateByCountPartitioner | package partitioner

class MongoPaginateByCountPartitioner extends Logging with MongoPartitioner with MongoPaginationPartitioner

Configuration Properties

Instance Constructors

new MongoPaginateByCountPartitioner()

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def calculatePartitions(connector: MongoConnector, readConfig: ReadConfig, partitionKey: String, count: Long, numDocumentsPerPartition: Int, matchQuery: Bson): Seq[BsonValue]

def clone(): AnyRef

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def getClass(): Class[_]

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logName: String

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

val numberOfPartitionsProperty: String

val partitionKeyProperty: String

def partitions(connector: MongoConnector, readConfig: ReadConfig, pipeline: Array[BsonDocument]): Array[MongoPartition]

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from MongoPaginationPartitioner

Inherited from MongoPartitioner

Inherited from Serializable

Inherited from Serializable

Inherited from Logging

Inherited from LoggingTrait

Inherited from AnyRef

Inherited from Any

Ungrouped