bigquery

Type Members

1. class ArrowBinaryIterator extends Iterator[InternalRow]

   

   An iterator for scanning over rows serialized in Arrow format

2. class ArrowSchemaConverter extends ColumnVector

   

3. class AvroBinaryIterator extends Iterator[InternalRow]

   

   An iterator for scanning over rows serialized in Avro format

4. implicit class BigQueryDataFrameReader extends AnyRef

   

5. implicit class BigQueryDataFrameWriter[T] extends AnyRef

   

6. case class BigQueryInsertableRelation(bigQuery: BigQuery, sqlContext: SQLContext, options: SparkBigQueryOptions) extends BaseRelation with InsertableRelation with Logging with Product with Serializable

   

   Used only to insert data into BigQuery.

   Used only to insert data into BigQuery. Getting the table metadata (existence, number of records, size, etc.) is done by API calls without reading existing data

7. class BigQueryRelationProvider extends RelationProvider with CreatableRelationProvider with SchemaRelationProvider with DataSourceRegister

   

8. case class BigQueryWriteHelper(bigQuery: BigQuery, sqlContext: SQLContext, saveMode: SaveMode, options: SparkBigQueryOptions, data: DataFrame, tableExists: Boolean) extends Logging with Product with Serializable

   

9. class DefaultSource extends BigQueryRelationProvider

   

10. case class IntermediateDataCleaner(path: Path, conf: Configuration) extends Thread with Logging with Product with Serializable

    

    Responsible for recursively deleting the intermediate path.

    Responsible for recursively deleting the intermediate path. Implementing Thread in order to act as shutdown hook.

    path

    the path to delete

    conf

    the hadoop configuration

11. case class SparkBigQueryOptions(tableId: TableId, parentProject: String, credentials: Option[String] = None, credentialsFile: Option[String] = None, filter: Option[String] = None, schema: Option[StructType] = None, maxParallelism: Option[Int] = None, temporaryGcsBucket: Option[String] = None, intermediateFormat: FormatOptions = SparkBigQueryOptions.DefaultFormat, readDataFormat: DataFormat = ..., combinePushedDownFilters: Boolean = true, viewsEnabled: Boolean = false, materializationProject: Option[String] = None, materializationDataset: Option[String] = None, partitionField: Option[String] = None, partitionExpirationMs: Option[Long] = None, partitionRequireFilter: Option[Boolean] = None, partitionType: Option[String] = None, clusteredFields: Option[Array[String]] = None, createDisposition: Option[CreateDisposition] = None, optimizedEmptyProjection: Boolean = true, accessToken: Option[String] = None, loadSchemaUpdateOptions: List[SchemaUpdateOption] = ImmutableList.of(), viewExpirationTimeInHours: Int = 24, maxReadRowsRetries: Int = 3) extends Product with Serializable

    

    Options for defining BigQueryRelations

12. case class ToIterator[E](remote: RemoteIterator[E]) extends Iterator[E] with Product with Serializable

    

    Converts HDFS RemoteIterator to Scala iterator