ReadConfig

Type Members

type Self = ReadConfig

The type of the MongoConfig
The type of the MongoConfig

Definition Classes
ReadConfig → MongoCompanionConfig

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def apply(options: Map[String, String], default: Option[ReadConfig]): ReadConfig

Create a configuration from the values in the Map, using the optional default configuration for any default values.
Create a configuration from the values in the Map, using the optional default configuration for any default values.
Note: Values in the map do not need to be prefixed with the configPrefix.
options
a map of properties and their string values
default
the optional default configuration, used for determining the default values for the properties
returns
the configuration

Definition Classes
ReadConfig → MongoCompanionConfig
def apply(options: Map[String, String]): Self

Create a configuration from the values in the Map
Create a configuration from the values in the Map
Note: Values in the map do not need to be prefixed with the configPrefix.
options
a map of properties and their string values
returns
the configuration

Definition Classes
MongoCompanionConfig
def apply(sparkConf: SparkConf, options: Map[String, String]): Self

Create a configuration from the sparkConf
Create a configuration from the sparkConf
Uses the prefixed properties that are set in the Spark configuration to create the config.
sparkConf
the spark configuration
options
overloaded parameters
returns
the configuration

Definition Classes
MongoCompanionConfig
See also
configPrefix
def apply(sparkConf: SparkConf): Self

Create a configuration from the sparkConf
Create a configuration from the sparkConf
Uses the prefixed properties that are set in the Spark configuration to create the config.
sparkConf
the spark configuration
returns
the configuration

Definition Classes
MongoCompanionConfig
See also
configPrefix
def apply(sparkSession: SparkSession): Self

Create a configuration from the sqlContext
Create a configuration from the sqlContext
Uses the prefixed properties that are set in the Spark configuration to create the config.
sparkSession
the SparkSession
returns
the configuration

Definition Classes
MongoCompanionConfig
See also
configPrefix
def apply(sparkContext: SparkContext): Self

Create a configuration from the sparkContext
Create a configuration from the sparkContext
Uses the prefixed properties that are set in the Spark configuration to create the config.
sparkContext
the spark context
returns
the configuration

Definition Classes
MongoCompanionConfig
See also
configPrefix
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def collectionName(collectionNameProperty: String, options: Map[String, String], default: Option[String] = None): String

Attributes
protected
Definition Classes
MongoCompanionConfig
val collectionNameProperty: String

The collection name property
The collection name property

Definition Classes
MongoInputConfig
val configPrefix: String

The configuration prefix string for the current configuration scope
The configuration prefix string for the current configuration scope

Definition Classes
MongoInputConfig → MongoCompanionConfig
def connectionString(options: Map[String, String]): ConnectionString

Attributes
protected
Definition Classes
MongoCompanionConfig
def create(sparkConf: SparkConf, options: Map[String, String]): ReadConfig

Create a configuration from the sparkConf
Create a configuration from the sparkConf
Uses the prefixed properties that are set in the Spark configuration to create the config.
sparkConf
the spark configuration
options
overloaded parameters
returns
the configuration

Definition Classes
ReadConfig → MongoCompanionConfig
See also
configPrefix
def create(options: Map[String, String], default: ReadConfig): ReadConfig

Create a configuration easily from the Java API using the values in the Map, using the optional default configuration for any default values.
Create a configuration easily from the Java API using the values in the Map, using the optional default configuration for any default values.
Note: Values in the map do not need to be prefixed with the configPrefix.
options
a map of properties and their string values
default
the optional default configuration, used for determining the default values for the properties
returns
the configuration

Definition Classes
ReadConfig → MongoCompanionConfig
def create(options: Map[String, String]): ReadConfig

Create a configuration easily from the Java API using the values in the Map
Create a configuration easily from the Java API using the values in the Map
Note: Values in the map do not need to be prefixed with the configPrefix.
options
a map of properties and their string values
returns
the configuration

Definition Classes
ReadConfig → MongoCompanionConfig
def create(sparkConf: SparkConf): ReadConfig

Create a configuration easily from the Java API using the sparkConf
Create a configuration easily from the Java API using the sparkConf
Uses the prefixed properties that are set in the Spark configuration to create the config.
sparkConf
the spark configuration
returns
the configuration

Definition Classes
ReadConfig → MongoCompanionConfig
See also
configPrefix
def create(sparkSession: SparkSession): ReadConfig

Create a configuration easily from the Java API using the JavaSparkContext
Create a configuration easily from the Java API using the JavaSparkContext
Uses the prefixed properties that are set in the Spark configuration to create the config.
sparkSession
the SparkSession
returns
the configuration

Definition Classes
ReadConfig → MongoCompanionConfig
See also
configPrefix
def create(javaSparkContext: JavaSparkContext): ReadConfig

Create a configuration easily from the Java API using the JavaSparkContext
Create a configuration easily from the Java API using the JavaSparkContext
Uses the prefixed properties that are set in the Spark configuration to create the config.
javaSparkContext
the java spark context
returns
the configuration

Definition Classes
ReadConfig → MongoCompanionConfig
See also
configPrefix
def create(databaseName: String, collectionName: String, connectionString: String, sampleSize: Int, partitioner: String, partitionerOptions: Map[String, String], localThreshold: Int, readPreference: ReadPreference, readConcern: ReadConcern, registerHelperFunctions: Boolean): ReadConfig

Read Configuration used when reading data from MongoDB
Read Configuration used when reading data from MongoDB
databaseName
the database name
collectionName
the collection name
connectionString
the optional connection string used in the creation of this configuration
sampleSize
a positive integer sample size to draw from the collection when inferring the schema
partitioner
the class name of the partitioner to use to create partitions
partitionerOptions
the configuration options for the partitioner
localThreshold
the local threshold in milliseconds used when choosing among multiple MongoDB servers to send a request. Only servers whose ping time is less than or equal to the server with the fastest ping time plus the local threshold will be chosen.
readPreference
the readPreference configuration
readConcern
the readConcern configuration

Since
1.0
def create(databaseName: String, collectionName: String, connectionString: String, sampleSize: Int, partitioner: String, partitionerOptions: Map[String, String], localThreshold: Int, readPreference: ReadPreference, readConcern: ReadConcern): ReadConfig

Read Configuration used when reading data from MongoDB
Read Configuration used when reading data from MongoDB
databaseName
the database name
collectionName
the collection name
connectionString
the optional connection string used in the creation of this configuration
sampleSize
a positive integer sample size to draw from the collection when inferring the schema
partitioner
the class name of the partitioner to use to create partitions
partitionerOptions
the configuration options for the partitioner
localThreshold
the local threshold in milliseconds used when choosing among multiple MongoDB servers to send a request. Only servers whose ping time is less than or equal to the server with the fastest ping time plus the local threshold will be chosen.
readPreference
the readPreference configuration
readConcern
the readConcern configuration

Since
1.0
def databaseName(databaseNameProperty: String, options: Map[String, String], default: Option[String] = None): String

Attributes
protected
Definition Classes
MongoCompanionConfig
val databaseNameProperty: String

The database name property
The database name property

Definition Classes
MongoInputConfig
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
def getBoolean(newValue: Option[String], existingValue: Option[Boolean] = None, defaultValue: Boolean): Boolean

Attributes
protected
Definition Classes
MongoCompanionConfig
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getInt(newValue: Option[String], existingValue: Option[Int] = None, defaultValue: Int): Int

Attributes
protected
Definition Classes
MongoCompanionConfig
def getOptionsFromConf(sparkConf: SparkConf): Map[String, String]

Gets an options map from the SparkConf
Gets an options map from the SparkConf
sparkConf
the SparkConf
returns
the options

Definition Classes
MongoCompanionConfig
def getString(newValue: Option[String], existingValue: Option[String] = None, defaultValue: String): String

Attributes
protected
Definition Classes
MongoCompanionConfig
def hashCode(): Int

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
val localThresholdProperty: String

The localThreshold property
The localThreshold property
The local threshold in milliseconds is used when choosing among multiple MongoDB servers to send a request. Only servers whose ping time is less than or equal to the server with the fastest ping time *plus* the local threshold will be chosen.
For example when choosing which MongoS to send a request through a localThreshold of 0 would pick the MongoS with the fastest ping time.
Default: 15 ms

Definition Classes
MongoInputConfig
def log: Logger

Attributes
protected
Definition Classes
LoggingTrait
def logDebug(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
LoggingTrait
def logDebug(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
LoggingTrait
def logError(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
LoggingTrait
def logError(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
LoggingTrait
def logInfo(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
LoggingTrait
def logInfo(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
LoggingTrait
def logName: String

Attributes
protected
Definition Classes
LoggingTrait
def logTrace(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
LoggingTrait
def logTrace(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
LoggingTrait
def logWarning(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
LoggingTrait
def logWarning(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
LoggingTrait
val mongoURIProperty: String

The mongo URI property
The mongo URI property
Represents a connection string.
Any values set in the connection string will override any default values for the configuration.

Definition Classes
MongoCompanionConfig
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
val partitionerOptionsProperty: String

The partitioner options property
The partitioner options property
Represents a map of options for customising the configuration of a partitioner. Default: Map.empty[String, String]

Definition Classes
MongoInputConfig
val partitionerProperty: String

The partition property
The partition property
Represents the name of the partitioner to use when partitioning the data in the collection. Default: MongoDefaultPartitioner

Definition Classes
MongoInputConfig
val readConcernLevelProperty: String

The ReadConcern level property
The ReadConcern level property
Default: DEFAULT

Definition Classes
MongoInputConfig
See also
ReadConcernConfig
val readPreferenceNameProperty: String

The ReadPreference name property
The ReadPreference name property
Default: primary

Definition Classes
MongoInputConfig
See also
ReadPreferenceConfig
val readPreferenceTagSetsProperty: String

The ReadPreference tags property
The ReadPreference tags property

Definition Classes
MongoInputConfig
See also
ReadPreferenceConfig
val registerSQLHelperFunctions: String

Register SQL Helper functions
Register SQL Helper functions
The SQL helper functions allow easy querying of Bson types inside SQL queries

Definition Classes
MongoInputConfig
Since
1.1
val sampleSizeProperty: String

The sample size property
The sample size property
Used when sampling data from MongoDB to determine the Schema. Default: 1000

Definition Classes
MongoInputConfig
def stripPrefix(options: Map[String, String]): Map[String, String]

Strip the prefix from options
Strip the prefix from options
options
options that may contain the prefix
returns
prefixLess options

Definition Classes
MongoCompanionConfig
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Deprecated Value Members

def apply(sqlContext: SQLContext): Self

Create a configuration from the sqlContext
Create a configuration from the sqlContext
Uses the prefixed properties that are set in the Spark configuration to create the config.
sqlContext
the SQL context
returns
the configuration

Definition Classes
MongoCompanionConfig
Annotations
@deprecated
Deprecated
(Since version 2.0.0) As of Spark 2.0 SQLContext was replaced by SparkSession. Use the SparkSession method instead
See also
configPrefix
def create(sqlContext: SQLContext): ReadConfig

Create a configuration easily from the Java API using the JavaSparkContext
Create a configuration easily from the Java API using the JavaSparkContext
Uses the prefixed properties that are set in the Spark configuration to create the config.
sqlContext
the SQL context
returns
the configuration

Definition Classes
ReadConfig → MongoCompanionConfig
Annotations
@deprecated
Deprecated
(Since version 2.0.0) As of Spark 2.0 SQLContext was replaced by SparkSession. Use the SparkSession method instead
See also
configPrefix

Related Docs: class ReadConfig | package config

object ReadConfig extends MongoInputConfig with LoggingTrait

Configuration Properties

Type Members

type Self = ReadConfig

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def apply(options: Map[String, String], default: Option[ReadConfig]): ReadConfig

def apply(options: Map[String, String]): Self

def apply(sparkConf: SparkConf, options: Map[String, String]): Self

def apply(sparkConf: SparkConf): Self

def apply(sparkSession: SparkSession): Self

def apply(sparkContext: SparkContext): Self

final def asInstanceOf[T0]: T0

def clone(): AnyRef

def collectionName(collectionNameProperty: String, options: Map[String, String], default: Option[String] = None): String

val collectionNameProperty: String

val configPrefix: String

def connectionString(options: Map[String, String]): ConnectionString

def create(sparkConf: SparkConf, options: Map[String, String]): ReadConfig

def create(options: Map[String, String], default: ReadConfig): ReadConfig

def create(options: Map[String, String]): ReadConfig

def create(sparkConf: SparkConf): ReadConfig

def create(sparkSession: SparkSession): ReadConfig

def create(javaSparkContext: JavaSparkContext): ReadConfig

def create(databaseName: String, collectionName: String, connectionString: String, sampleSize: Int, partitioner: String, partitionerOptions: Map[String, String], localThreshold: Int, readPreference: ReadPreference, readConcern: ReadConcern, registerHelperFunctions: Boolean): ReadConfig

def create(databaseName: String, collectionName: String, connectionString: String, sampleSize: Int, partitioner: String, partitionerOptions: Map[String, String], localThreshold: Int, readPreference: ReadPreference, readConcern: ReadConcern): ReadConfig

def databaseName(databaseNameProperty: String, options: Map[String, String], default: Option[String] = None): String

val databaseNameProperty: String

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

def getBoolean(newValue: Option[String], existingValue: Option[Boolean] = None, defaultValue: Boolean): Boolean

final def getClass(): Class[_]

def getInt(newValue: Option[String], existingValue: Option[Int] = None, defaultValue: Int): Int

def getOptionsFromConf(sparkConf: SparkConf): Map[String, String]

def getString(newValue: Option[String], existingValue: Option[String] = None, defaultValue: String): String

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

val localThresholdProperty: String

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logName: String

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

val mongoURIProperty: String

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

val partitionerOptionsProperty: String

val partitionerProperty: String

val readConcernLevelProperty: String

val readPreferenceNameProperty: String

val readPreferenceTagSetsProperty: String

val registerSQLHelperFunctions: String

val sampleSizeProperty: String

def stripPrefix(options: Map[String, String]): Map[String, String]

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Deprecated Value Members

def apply(sqlContext: SQLContext): Self

def create(sqlContext: SQLContext): ReadConfig

Inherited from LoggingTrait

Inherited from MongoInputConfig

Inherited from MongoCompanionConfig

Inherited from Serializable

Inherited from Serializable