public abstract static class BigQueryIO.TypedRead<T>
extends org.apache.beam.sdk.transforms.PTransform<org.apache.beam.sdk.values.PBegin,org.apache.beam.sdk.values.PCollection<T>>
implements org.apache.beam.sdk.schemas.ProjectionProducer<org.apache.beam.sdk.transforms.PTransform<org.apache.beam.sdk.values.PBegin,org.apache.beam.sdk.values.PCollection<T>>>
BigQueryIO.read(SerializableFunction)
.Modifier and Type | Class and Description |
---|---|
static class |
BigQueryIO.TypedRead.Method
Determines the method used to read data from BigQuery.
|
static class |
BigQueryIO.TypedRead.QueryPriority
An enumeration type for the priority of a query.
|
Constructor and Description |
---|
TypedRead() |
Modifier and Type | Method and Description |
---|---|
org.apache.beam.sdk.transforms.PTransform<org.apache.beam.sdk.values.PBegin,org.apache.beam.sdk.values.PCollection<T>> |
actuateProjectionPushdown(java.util.Map<org.apache.beam.sdk.values.TupleTag<?>,org.apache.beam.sdk.schemas.FieldAccessDescriptor> outputFields) |
org.apache.beam.sdk.values.PCollection<T> |
expand(org.apache.beam.sdk.values.PBegin input) |
BigQueryIO.TypedRead<T> |
from(java.lang.String tableSpec)
|
BigQueryIO.TypedRead<T> |
from(com.google.api.services.bigquery.model.TableReference table)
|
BigQueryIO.TypedRead<T> |
from(org.apache.beam.sdk.options.ValueProvider<java.lang.String> tableSpec)
|
BigQueryIO.TypedRead<T> |
fromQuery(java.lang.String query)
|
BigQueryIO.TypedRead<T> |
fromQuery(org.apache.beam.sdk.options.ValueProvider<java.lang.String> query)
|
abstract BigQueryIO.TypedRead.Method |
getMethod() |
@Nullable com.google.api.services.bigquery.model.TableReference |
getTable()
|
@Nullable org.apache.beam.sdk.options.ValueProvider<com.google.api.services.bigquery.model.TableReference> |
getTableProvider()
|
void |
populateDisplayData(org.apache.beam.sdk.transforms.display.DisplayData.Builder builder) |
boolean |
supportsProjectionPushdown() |
BigQueryIO.TypedRead<T> |
useAvroLogicalTypes() |
BigQueryIO.TypedRead<T> |
usingStandardSql()
|
void |
validate(org.apache.beam.sdk.options.PipelineOptions options) |
BigQueryIO.TypedRead<T> |
withBeamRowConverters(org.apache.beam.sdk.values.TypeDescriptor<T> typeDescriptor,
org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.TypedRead.ToBeamRowFunction<T> toRowFn,
org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.TypedRead.FromBeamRowFunction<T> fromRowFn)
Sets the functions to convert elements to/from
Row objects. |
BigQueryIO.TypedRead<T> |
withCoder(org.apache.beam.sdk.coders.Coder<T> coder)
Sets a
Coder for the result of the parse function. |
BigQueryIO.TypedRead<T> |
withFormat(com.google.cloud.bigquery.storage.v1.DataFormat format)
See
DataFormat . |
BigQueryIO.TypedRead<T> |
withKmsKey(java.lang.String kmsKey)
For query sources, use this Cloud KMS key to encrypt any temporary tables created.
|
BigQueryIO.TypedRead<T> |
withMethod(BigQueryIO.TypedRead.Method method)
|
BigQueryIO.TypedRead<T> |
withoutResultFlattening()
|
BigQueryIO.TypedRead<T> |
withoutValidation()
|
BigQueryIO.TypedRead<T> |
withQueryLocation(java.lang.String location)
BigQuery geographic location where the query job will be
executed.
|
BigQueryIO.TypedRead<T> |
withQueryPriority(BigQueryIO.TypedRead.QueryPriority priority)
|
BigQueryIO.TypedRead<T> |
withQueryTempDataset(java.lang.String queryTempDatasetRef)
Temporary dataset reference when using
fromQuery(String) . |
BigQueryIO.TypedRead<T> |
withRowRestriction(java.lang.String rowRestriction)
|
BigQueryIO.TypedRead<T> |
withRowRestriction(org.apache.beam.sdk.options.ValueProvider<java.lang.String> rowRestriction)
Read only rows which match the specified filter, which must be a SQL expression compatible
with Google standard
SQL.
|
BigQueryIO.TypedRead<T> |
withSelectedFields(java.util.List<java.lang.String> selectedFields)
|
BigQueryIO.TypedRead<T> |
withSelectedFields(org.apache.beam.sdk.options.ValueProvider<java.util.List<java.lang.String>> selectedFields)
Read only the specified fields (columns) from a BigQuery table.
|
BigQueryIO.TypedRead<T> |
withTemplateCompatibility() |
BigQueryIO.TypedRead<T> |
withTestServices(BigQueryServices testServices) |
public abstract BigQueryIO.TypedRead.Method getMethod()
public void validate(org.apache.beam.sdk.options.PipelineOptions options)
validate
in class org.apache.beam.sdk.transforms.PTransform<org.apache.beam.sdk.values.PBegin,org.apache.beam.sdk.values.PCollection<T>>
public org.apache.beam.sdk.values.PCollection<T> expand(org.apache.beam.sdk.values.PBegin input)
expand
in class org.apache.beam.sdk.transforms.PTransform<org.apache.beam.sdk.values.PBegin,org.apache.beam.sdk.values.PCollection<T>>
public void populateDisplayData(org.apache.beam.sdk.transforms.display.DisplayData.Builder builder)
populateDisplayData
in interface org.apache.beam.sdk.transforms.display.HasDisplayData
populateDisplayData
in class org.apache.beam.sdk.transforms.PTransform<org.apache.beam.sdk.values.PBegin,org.apache.beam.sdk.values.PCollection<T>>
public @Nullable org.apache.beam.sdk.options.ValueProvider<com.google.api.services.bigquery.model.TableReference> getTableProvider()
public @Nullable com.google.api.services.bigquery.model.TableReference getTable()
public BigQueryIO.TypedRead<T> withCoder(org.apache.beam.sdk.coders.Coder<T> coder)
Coder
for the result of the parse function. This may be required if a coder
can not be inferred automatically.public BigQueryIO.TypedRead<T> withKmsKey(java.lang.String kmsKey)
public BigQueryIO.TypedRead<T> withBeamRowConverters(org.apache.beam.sdk.values.TypeDescriptor<T> typeDescriptor, org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.TypedRead.ToBeamRowFunction<T> toRowFn, org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.TypedRead.FromBeamRowFunction<T> fromRowFn)
Row
objects.
Setting these conversion functions is necessary to enable Schema
support.
public BigQueryIO.TypedRead<T> from(java.lang.String tableSpec)
public BigQueryIO.TypedRead<T> from(org.apache.beam.sdk.options.ValueProvider<java.lang.String> tableSpec)
public BigQueryIO.TypedRead<T> fromQuery(java.lang.String query)
public BigQueryIO.TypedRead<T> fromQuery(org.apache.beam.sdk.options.ValueProvider<java.lang.String> query)
public BigQueryIO.TypedRead<T> from(com.google.api.services.bigquery.model.TableReference table)
public BigQueryIO.TypedRead<T> withoutValidation()
public BigQueryIO.TypedRead<T> withoutResultFlattening()
public BigQueryIO.TypedRead<T> usingStandardSql()
public BigQueryIO.TypedRead<T> withQueryPriority(BigQueryIO.TypedRead.QueryPriority priority)
public BigQueryIO.TypedRead<T> withQueryLocation(java.lang.String location)
public BigQueryIO.TypedRead<T> withQueryTempDataset(java.lang.String queryTempDatasetRef)
fromQuery(String)
. When reading from a query,
BigQuery will create a temporary dataset and a temporary table to store the results of the
query. With this option, you can set an existing dataset to create the temporary table.
BigQueryIO will create a temporary table in that dataset, and will remove it once it is not
needed. No other tables in the dataset will be modified. If your job does not have
permissions to create a new dataset, and you want to use fromQuery(String)
(for
instance, to read from a view), you should use this option. Remember that the dataset must
exist and your job needs permissions to create and remove tables inside that dataset.public BigQueryIO.TypedRead<T> withMethod(BigQueryIO.TypedRead.Method method)
public BigQueryIO.TypedRead<T> withFormat(com.google.cloud.bigquery.storage.v1.DataFormat format)
DataFormat
.public BigQueryIO.TypedRead<T> withSelectedFields(java.util.List<java.lang.String> selectedFields)
public BigQueryIO.TypedRead<T> withSelectedFields(org.apache.beam.sdk.options.ValueProvider<java.util.List<java.lang.String>> selectedFields)
Requires BigQueryIO.TypedRead.Method.DIRECT_READ
. Not compatible with fromQuery(String)
.
public BigQueryIO.TypedRead<T> withRowRestriction(java.lang.String rowRestriction)
public BigQueryIO.TypedRead<T> withRowRestriction(org.apache.beam.sdk.options.ValueProvider<java.lang.String> rowRestriction)
Requires BigQueryIO.TypedRead.Method.DIRECT_READ
. Not compatible with fromQuery(String)
.
public BigQueryIO.TypedRead<T> withTemplateCompatibility()
public BigQueryIO.TypedRead<T> withTestServices(BigQueryServices testServices)
public BigQueryIO.TypedRead<T> useAvroLogicalTypes()
public boolean supportsProjectionPushdown()
supportsProjectionPushdown
in interface org.apache.beam.sdk.schemas.ProjectionProducer<org.apache.beam.sdk.transforms.PTransform<org.apache.beam.sdk.values.PBegin,org.apache.beam.sdk.values.PCollection<T>>>
public org.apache.beam.sdk.transforms.PTransform<org.apache.beam.sdk.values.PBegin,org.apache.beam.sdk.values.PCollection<T>> actuateProjectionPushdown(java.util.Map<org.apache.beam.sdk.values.TupleTag<?>,org.apache.beam.sdk.schemas.FieldAccessDescriptor> outputFields)
actuateProjectionPushdown
in interface org.apache.beam.sdk.schemas.ProjectionProducer<org.apache.beam.sdk.transforms.PTransform<org.apache.beam.sdk.values.PBegin,org.apache.beam.sdk.values.PCollection<T>>>