JdbcUtils

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def createConnectionFactory(options: JDBCOptions): () ⇒ Connection

Returns a factory for creating connections to the given JDBC URL.
Returns a factory for creating connections to the given JDBC URL.
options
- JDBC options that contains url, table and other information.
def createTable(conn: Connection, schema: StructType, sparkSession: SparkSession, options: JDBCOptions): Unit

Creates a table with a given schema.
def dropTable(conn: Connection, table: String): Unit

Drops a table from the JDBC database.
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def executeSimpleStatement(getConnection: () ⇒ Connection, statement: String, isolationLevel: Int): Iterator[Byte]
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getCommonJDBCType(dt: DataType): Option[JdbcType]

Retrieve standard jdbc types.
Retrieve standard jdbc types.
dt
The datatype (e.g. org.apache.spark.sql.types.StringType)
returns
The default JdbcType for this DataType
def getCopyStatement(sourceTable: String, targetTable: String, rddSchema: StructType, tableSchema: Option[StructType], dialect: JdbcDialect): String

Returns an Insert SQL statement for copying all rows from the source table into the target table via JDBC conn.
def getCustomSchema(tableSchema: StructType, customSchema: String, nameEquality: Resolver): StructType

Parses the user specified customSchema option value to DataFrame schema, and returns a schema that is replaced by the custom schema's dataType if column name is matched.
def getInsertStatement(table: String, rddSchema: StructType, tableSchema: Option[StructType], isCaseSensitive: Boolean, dialect: JdbcDialect): String

Returns an Insert SQL statement for inserting a row into the target table via JDBC conn.
def getJdbcType(dt: DataType, dialect: JdbcDialect): JdbcType
def getSchema(resultSet: ResultSet, dialect: JdbcDialect, alwaysNullable: Boolean = false): StructType

Takes a ResultSet and returns its Catalyst schema.
Takes a ResultSet and returns its Catalyst schema.
alwaysNullable
If true, all the columns are nullable.
returns
A StructType giving the Catalyst schema.

Exceptions thrown
SQLException if the schema contains an unsupported type.
def getSchemaOption(conn: Connection, options: JDBCOptions): Option[StructType]

Returns the schema if the table already exists in the JDBC database.
def getStatement(conf: RuntimeConfig, table: String, rddSchema: StructType, tableSchema: Option[StructType], isCaseSensitive: Boolean, dialect: JdbcDialect): String
def getTableName(options: JDBCOptions): String
def getTruncateStatement(table: String): String

Returns an truncate SQL statement for truncating a table via JDBC conn.
def hashCode(): Int

Definition Classes
AnyRef → Any
def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean

Attributes
protected
Definition Classes
Logging
def initializeLogIfNecessary(isInterpreter: Boolean): Unit

Attributes
protected
Definition Classes
Logging
def isCascadingTruncateTable(url: String): Option[Boolean]
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def isTableEmpty(conn: Connection, options: JDBCOptions): Boolean

Returns true if the table is empty
def isTraceEnabled(): Boolean

Attributes
protected
Definition Classes
Logging
def log: Logger

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logName: String

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def makeInternalSetter(conn: Connection, dialect: JdbcDialect, dataType: DataType): JDBCInternalValueSetter
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def resultSetToRows(resultSet: ResultSet, schema: StructType): Iterator[Row]

Convert a ResultSet into an iterator of Catalyst Rows.
def saveInternalPartition(getConnection: () ⇒ Connection, table: String, iterator: Iterator[InternalRow], rddSchema: StructType, insertStmt: String, batchSize: Int, dialect: JdbcDialect, isolationLevel: Int): Iterator[Byte]

Saves a partition of a DataFrame to the JDBC database.
Saves a partition of a DataFrame to the JDBC database. This is done in a single database transaction (unless isolation level is "NONE") in order to avoid repeatedly inserting data as much as possible.
It is still theoretically possible for rows in a DataFrame to be inserted into the database more than once if a stage somehow fails after the commit occurs but before the stage can return successfully.
This is not a closure inside saveTable() because apparently cosmetic implementation changes elsewhere might easily render such a closure non-Serializable. Instead, we explicitly close over all variables that are used.
def savePartition(getConnection: () ⇒ Connection, table: String, iterator: Iterator[Row], rddSchema: StructType, insertStmt: String, batchSize: Int, dialect: JdbcDialect, isolationLevel: Int): Iterator[Byte]

Saves a partition of a DataFrame to the JDBC database.
Saves a partition of a DataFrame to the JDBC database. This is done in a single database transaction (unless isolation level is "NONE") in order to avoid repeatedly inserting data as much as possible.
It is still theoretically possible for rows in a DataFrame to be inserted into the database more than once if a stage somehow fails after the commit occurs but before the stage can return successfully.
This is not a closure inside saveTable() because apparently cosmetic implementation changes elsewhere might easily render such a closure non-Serializable. Instead, we explicitly close over all variables that are used.
def saveTable(df: DataFrame, tableSchema: Option[StructType], isCaseSensitive: Boolean, options: JDBCOptions): Unit

Saves the RDD to the database in a single transaction.
def schemaString(schema: StructType, sparkSession: SparkSession, url: String, createTableColumnTypes: Option[String] = None): String

Compute the schema string for this RDD.
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def tableExists(conn: Connection, options: JDBCOptions): Boolean

Returns true if the table already exists in the JDBC database.
def toString(): String

Definition Classes
AnyRef → Any
def truncateTable(conn: Connection, options: JDBCOptions): Unit

Truncates a table from the JDBC database without side effects.
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Doc: package jdbc2

object JdbcUtils extends Logging

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def clone(): AnyRef

def createConnectionFactory(options: JDBCOptions): () ⇒ Connection

def createTable(conn: Connection, schema: StructType, sparkSession: SparkSession, options: JDBCOptions): Unit

def dropTable(conn: Connection, table: String): Unit

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def executeSimpleStatement(getConnection: () ⇒ Connection, statement: String, isolationLevel: Int): Iterator[Byte]

def finalize(): Unit

final def getClass(): Class[_]

def getCommonJDBCType(dt: DataType): Option[JdbcType]

def getCopyStatement(sourceTable: String, targetTable: String, rddSchema: StructType, tableSchema: Option[StructType], dialect: JdbcDialect): String

def getCustomSchema(tableSchema: StructType, customSchema: String, nameEquality: Resolver): StructType

def getInsertStatement(table: String, rddSchema: StructType, tableSchema: Option[StructType], isCaseSensitive: Boolean, dialect: JdbcDialect): String

def getJdbcType(dt: DataType, dialect: JdbcDialect): JdbcType

def getSchema(resultSet: ResultSet, dialect: JdbcDialect, alwaysNullable: Boolean = false): StructType

def getSchemaOption(conn: Connection, options: JDBCOptions): Option[StructType]

def getStatement(conf: RuntimeConfig, table: String, rddSchema: StructType, tableSchema: Option[StructType], isCaseSensitive: Boolean, dialect: JdbcDialect): String

def getTableName(options: JDBCOptions): String

def getTruncateStatement(table: String): String

def hashCode(): Int

def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean

def initializeLogIfNecessary(isInterpreter: Boolean): Unit

def isCascadingTruncateTable(url: String): Option[Boolean]

final def isInstanceOf[T0]: Boolean

def isTableEmpty(conn: Connection, options: JDBCOptions): Boolean

def isTraceEnabled(): Boolean

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logName: String

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

def makeInternalSetter(conn: Connection, dialect: JdbcDialect, dataType: DataType): JDBCInternalValueSetter

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def resultSetToRows(resultSet: ResultSet, schema: StructType): Iterator[Row]

def saveInternalPartition(getConnection: () ⇒ Connection, table: String, iterator: Iterator[InternalRow], rddSchema: StructType, insertStmt: String, batchSize: Int, dialect: JdbcDialect, isolationLevel: Int): Iterator[Byte]

def savePartition(getConnection: () ⇒ Connection, table: String, iterator: Iterator[Row], rddSchema: StructType, insertStmt: String, batchSize: Int, dialect: JdbcDialect, isolationLevel: Int): Iterator[Byte]

def saveTable(df: DataFrame, tableSchema: Option[StructType], isCaseSensitive: Boolean, options: JDBCOptions): Unit

def schemaString(schema: StructType, sparkSession: SparkSession, url: String, createTableColumnTypes: Option[String] = None): String

final def synchronized[T0](arg0: ⇒ T0): T0

def tableExists(conn: Connection, options: JDBCOptions): Boolean

def toString(): String

def truncateTable(conn: Connection, options: JDBCOptions): Unit

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from Logging

Inherited from AnyRef

Inherited from Any

Ungrouped