Skip to content
Permalink
Browse files
feat: add support for DecimalTargetTypes (#1345)
* feat: add support for DecimalTargetType

Towards b/188414733
Implemented in LoadJobConfiguration

* update test to verify that the destination table's decimal column is loaded as BIGNUMERIC type

* add decimalTargetTypes support in ExternalTableDefinition.java

* add decimalTargetTypes support in ExternalTableDefinition.java
  • Loading branch information
stephaniewang526 committed Jun 8, 2021
1 parent 5f6307f commit ba528df03def71907e2811cf267718f090605d95
@@ -4,32 +4,12 @@
<!-- TODO: REMOVE AFTER RELEASE -->
<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/bigquery/MaterializedViewDefinition</className>
<method>com.google.cloud.bigquery.Clustering getClustering()</method>
<className>com/google/cloud/bigquery/ExternalTableDefinition</className>
<method>com.google.common.collect.ImmutableList getDecimalTargetTypes()</method>
</difference>
<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/bigquery/MaterializedViewDefinition</className>
<method>com.google.cloud.bigquery.RangePartitioning getRangePartitioning()</method>
</difference>
<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/bigquery/MaterializedViewDefinition</className>
<method>com.google.cloud.bigquery.TimePartitioning getTimePartitioning()</method>
</difference>
<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/bigquery/MaterializedViewDefinition$Builder</className>
<method>com.google.cloud.bigquery.MaterializedViewDefinition$Builder setClustering(com.google.cloud.bigquery.Clustering)</method>
</difference>
<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/bigquery/MaterializedViewDefinition$Builder</className>
<method>com.google.cloud.bigquery.MaterializedViewDefinition$Builder setRangePartitioning(com.google.cloud.bigquery.RangePartitioning)</method>
</difference>
<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/bigquery/MaterializedViewDefinition$Builder</className>
<method>com.google.cloud.bigquery.MaterializedViewDefinition$Builder setTimePartitioning(com.google.cloud.bigquery.TimePartitioning)</method>
<className>com/google/cloud/bigquery/ExternalTableDefinition$Builder</className>
<method> com.google.cloud.bigquery.ExternalTableDefinition$Builder setDecimalTargetTypes(java.util.List)</method>
</difference>
</differences>
@@ -93,6 +93,17 @@ public Builder setFormatOptions(FormatOptions formatOptions) {
return setFormatOptionsInner(formatOptions);
}

/**
* Defines the list of possible SQL data types to which the source decimal values are converted.
* This list and the precision and the scale parameters of the decimal field determine the
* target type. In the order of NUMERIC, BIGNUMERIC, and STRING, a type is picked if it is in
* the specified list and if it supports the precision and the scale. STRING supports all
* precision and scale values.
*
* @param decimalTargetTypes decimalTargetType or {@code null} for none
*/
public abstract Builder setDecimalTargetTypes(List<String> decimalTargetTypes);

abstract Builder setFormatOptionsInner(FormatOptions formatOptions);

/**
@@ -229,6 +240,9 @@ public <F extends FormatOptions> F getFormatOptions() {
@Nullable
abstract FormatOptions getFormatOptionsInner();

@Nullable
public abstract ImmutableList<String> getDecimalTargetTypes();

/**
* [Experimental] Returns whether automatic detection of schema and format options should be
* performed.
@@ -283,6 +297,9 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toExternalDataC
if (getSourceUris() != null) {
externalConfigurationPb.setSourceUris(getSourceUris());
}
if (getDecimalTargetTypes() != null) {
externalConfigurationPb.setDecimalTargetTypes(getDecimalTargetTypes());
}
if (getFormatOptions() != null && FormatOptions.CSV.equals(getFormatOptions().getType())) {
externalConfigurationPb.setCsvOptions(((CsvOptions) getFormatOptions()).toPb());
}
@@ -430,6 +447,10 @@ static ExternalTableDefinition fromPb(Table tablePb) {
if (externalDataConfiguration.getSourceUris() != null) {
builder.setSourceUris(ImmutableList.copyOf(externalDataConfiguration.getSourceUris()));
}
if (externalDataConfiguration.getDecimalTargetTypes() != null) {
builder.setDecimalTargetTypes(
ImmutableList.copyOf(externalDataConfiguration.getDecimalTargetTypes()));
}
if (externalDataConfiguration.getSourceFormat() != null) {
builder.setFormatOptions(FormatOptions.of(externalDataConfiguration.getSourceFormat()));
}
@@ -469,6 +490,9 @@ static ExternalTableDefinition fromExternalDataConfiguration(
if (externalDataConfiguration.getSourceUris() != null) {
builder.setSourceUris(externalDataConfiguration.getSourceUris());
}
if (externalDataConfiguration.getDecimalTargetTypes() != null) {
builder.setDecimalTargetTypes(externalDataConfiguration.getDecimalTargetTypes());
}
if (externalDataConfiguration.getSchema() != null) {
builder.setSchema(Schema.fromPb(externalDataConfiguration.getSchema()));
}
@@ -38,6 +38,7 @@ public final class LoadJobConfiguration extends JobConfiguration implements Load

private final List<String> sourceUris;
private final TableId destinationTable;
private final List<String> decimalTargetTypes;
private final EncryptionConfiguration destinationEncryptionConfiguration;
private final JobInfo.CreateDisposition createDisposition;
private final JobInfo.WriteDisposition writeDisposition;
@@ -61,6 +62,7 @@ public static final class Builder extends JobConfiguration.Builder<LoadJobConfig

private List<String> sourceUris;
private TableId destinationTable;
private List<String> decimalTargetTypes;
private EncryptionConfiguration destinationEncryptionConfiguration;
private JobInfo.CreateDisposition createDisposition;
private JobInfo.WriteDisposition writeDisposition;
@@ -87,6 +89,7 @@ private Builder() {
private Builder(LoadJobConfiguration loadConfiguration) {
this();
this.destinationTable = loadConfiguration.destinationTable;
this.decimalTargetTypes = loadConfiguration.decimalTargetTypes;
this.createDisposition = loadConfiguration.createDisposition;
this.writeDisposition = loadConfiguration.writeDisposition;
this.formatOptions = loadConfiguration.formatOptions;
@@ -112,6 +115,9 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur
this();
JobConfigurationLoad loadConfigurationPb = configurationPb.getLoad();
this.destinationTable = TableId.fromPb(loadConfigurationPb.getDestinationTable());
if (loadConfigurationPb.getDecimalTargetTypes() != null) {
this.decimalTargetTypes = ImmutableList.copyOf(loadConfigurationPb.getDecimalTargetTypes());
}
if (loadConfigurationPb.getCreateDisposition() != null) {
this.createDisposition =
JobInfo.CreateDisposition.valueOf(loadConfigurationPb.getCreateDisposition());
@@ -278,6 +284,20 @@ public Builder setSourceUris(List<String> sourceUris) {
return this;
}

/**
* Defines the list of possible SQL data types to which the source decimal values are converted.
* This list and the precision and the scale parameters of the decimal field determine the
* target type. In the order of NUMERIC, BIGNUMERIC, and STRING, a type is picked if it is in
* the specified list and if it supports the precision and the scale. STRING supports all
* precision and scale values.
*
* @param decimalTargetTypes decimalTargetType or {@code null} for none
*/
public Builder setDecimalTargetTypes(List<String> decimalTargetTypes) {
this.decimalTargetTypes = decimalTargetTypes;
return this;
}

public Builder setAutodetect(Boolean autodetect) {
this.autodetect = autodetect;
return this;
@@ -341,6 +361,7 @@ private LoadJobConfiguration(Builder builder) {
super(builder);
this.sourceUris = builder.sourceUris;
this.destinationTable = builder.destinationTable;
this.decimalTargetTypes = builder.decimalTargetTypes;
this.createDisposition = builder.createDisposition;
this.writeDisposition = builder.writeDisposition;
this.formatOptions = builder.formatOptions;
@@ -430,6 +451,10 @@ public List<String> getSourceUris() {
return sourceUris;
}

public List<String> getDecimalTargetTypes() {
return decimalTargetTypes;
}

public Boolean getAutodetect() {
return autodetect;
}
@@ -482,6 +507,7 @@ public Builder toBuilder() {
ToStringHelper toStringHelper() {
return super.toStringHelper()
.add("destinationTable", destinationTable)
.add("decimalTargetTypes", decimalTargetTypes)
.add("destinationEncryptionConfiguration", destinationEncryptionConfiguration)
.add("createDisposition", createDisposition)
.add("writeDisposition", writeDisposition)
@@ -568,6 +594,9 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() {
if (sourceUris != null) {
loadConfigurationPb.setSourceUris(ImmutableList.copyOf(sourceUris));
}
if (decimalTargetTypes != null) {
loadConfigurationPb.setDecimalTargetTypes(ImmutableList.copyOf(decimalTargetTypes));
}
if (schemaUpdateOptions != null) {
ImmutableList.Builder<String> schemaUpdateOptionsBuilder = new ImmutableList.Builder<>();
for (JobInfo.SchemaUpdateOption schemaUpdateOption : schemaUpdateOptions) {
@@ -27,6 +27,8 @@
public class ExternalTableDefinitionTest {

private static final List<String> SOURCE_URIS = ImmutableList.of("uri1", "uri2");
private static final List<String> DECIMAL_TARGET_TYPES =
ImmutableList.of("NUMERIC", "BIGNUMERIC", "STRING");
private static final Field FIELD_SCHEMA1 =
Field.newBuilder("StringField", LegacySQLTypeName.STRING)
.setMode(Field.Mode.NULLABLE)
@@ -56,6 +58,7 @@ public class ExternalTableDefinitionTest {
.build();
private static final ExternalTableDefinition EXTERNAL_TABLE_DEFINITION =
ExternalTableDefinition.newBuilder(SOURCE_URIS, TABLE_SCHEMA, CSV_OPTIONS)
.setDecimalTargetTypes(DECIMAL_TARGET_TYPES)
.setCompression(COMPRESSION)
.setConnectionId(CONNECTION_ID)
.setIgnoreUnknownValues(IGNORE_UNKNOWN_VALUES)
@@ -111,6 +114,7 @@ public void testBuilder() {
assertEquals(MAX_BAD_RECORDS, EXTERNAL_TABLE_DEFINITION.getMaxBadRecords());
assertEquals(TABLE_SCHEMA, EXTERNAL_TABLE_DEFINITION.getSchema());
assertEquals(SOURCE_URIS, EXTERNAL_TABLE_DEFINITION.getSourceUris());
assertEquals(DECIMAL_TARGET_TYPES, EXTERNAL_TABLE_DEFINITION.getDecimalTargetTypes());
assertEquals(AUTODETECT, EXTERNAL_TABLE_DEFINITION.getAutodetect());
assertEquals(HIVE_PARTITIONING_OPTIONS, EXTERNAL_TABLE_DEFINITION.getHivePartitioningOptions());
assertNotEquals(EXTERNAL_TABLE_DEFINITION, TableDefinition.Type.EXTERNAL);
@@ -130,6 +134,7 @@ public void testToAndFromPb() {
private void compareExternalTableDefinition(
ExternalTableDefinition expected, ExternalTableDefinition value) {
assertEquals(expected, value);
assertEquals(expected.getDecimalTargetTypes(), value.getDecimalTargetTypes());
assertEquals(expected.getCompression(), value.getCompression());
assertEquals(expected.getConnectionId(), value.getConnectionId());
assertEquals(expected.getFormatOptions(), value.getFormatOptions());
@@ -50,6 +50,8 @@ public class LoadJobConfigurationTest {
.setDescription("FieldDescription")
.build();
private static final List<String> SOURCE_URIS = ImmutableList.of("uri1", "uri2");
private static final List<String> DECIMAL_TARGET_TYPES =
ImmutableList.of("NUMERIC", "BIGNUMERIC", "STRING");
private static final List<SchemaUpdateOption> SCHEMA_UPDATE_OPTIONS =
ImmutableList.of(SchemaUpdateOption.ALLOW_FIELD_ADDITION);
private static final Schema TABLE_SCHEMA = Schema.of(FIELD_SCHEMA);
@@ -76,6 +78,7 @@ public class LoadJobConfigurationTest {
.build();
private static final LoadJobConfiguration LOAD_CONFIGURATION_CSV =
LoadJobConfiguration.newBuilder(TABLE_ID, SOURCE_URIS)
.setDecimalTargetTypes(DECIMAL_TARGET_TYPES)
.setCreateDisposition(CREATE_DISPOSITION)
.setWriteDisposition(WRITE_DISPOSITION)
.setFormatOptions(CSV_OPTIONS)
@@ -228,6 +231,7 @@ private void compareLoadJobConfiguration(
assertEquals(expected.hashCode(), value.hashCode());
assertEquals(expected.toString(), value.toString());
assertEquals(expected.getDestinationTable(), value.getDestinationTable());
assertEquals(expected.getDecimalTargetTypes(), value.getDecimalTargetTypes());
assertEquals(expected.getCreateDisposition(), value.getCreateDisposition());
assertEquals(expected.getWriteDisposition(), value.getWriteDisposition());
assertEquals(expected.getCsvOptions(), value.getCsvOptions());
@@ -2747,6 +2747,54 @@ public void testLoadJobWithRangePartitioning() throws InterruptedException {
}
}

@Test
public void testLoadJobWithDecimalTargetTypes() throws InterruptedException {
String tableName = "test_load_job_table_parquet_decimalTargetTypes";
TableId destinationTable = TableId.of(DATASET, tableName);
String sourceUri = "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/numeric/numeric_38_12.parquet";
try {
LoadJobConfiguration configuration =
LoadJobConfiguration.newBuilder(destinationTable, sourceUri, FormatOptions.parquet())
.setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED)
.setDecimalTargetTypes(ImmutableList.of("NUMERIC", "BIGNUMERIC", "STRING"))
.build();
Job job = bigquery.create(JobInfo.of(configuration));
job = job.waitFor();
assertNull(job.getStatus().getError());
LoadJobConfiguration loadJobConfiguration = job.getConfiguration();
assertEquals(
ImmutableList.of("NUMERIC", "BIGNUMERIC", "STRING"),
loadJobConfiguration.getDecimalTargetTypes());
Table remoteTable = bigquery.getTable(DATASET, tableName);
assertNotNull(remoteTable);
assertEquals(
remoteTable.getDefinition().getSchema().getFields().get(0).getType().toString(),
"BIGNUMERIC");
} finally {
bigquery.delete(destinationTable);
}
}

@Test
public void testExternalTableWithDecimalTargetTypes() throws InterruptedException {
String tableName = "test_create_external_table_parquet_decimalTargetTypes";
TableId destinationTable = TableId.of(DATASET, tableName);
String sourceUri = "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/numeric/numeric_38_12.parquet";
ExternalTableDefinition externalTableDefinition =
ExternalTableDefinition.newBuilder(sourceUri, FormatOptions.parquet())
.setDecimalTargetTypes(ImmutableList.of("NUMERIC", "BIGNUMERIC", "STRING"))
.build();
TableInfo tableInfo = TableInfo.of(destinationTable, externalTableDefinition);
Table createdTable = bigquery.create(tableInfo);
assertNotNull(createdTable);
Table remoteTable = bigquery.getTable(DATASET, tableName);
assertNotNull(remoteTable);
assertEquals(
remoteTable.getDefinition().getSchema().getFields().get(0).getType().toString(),
"BIGNUMERIC");
assertTrue(remoteTable.delete());
}

@Test
public void testQueryJobWithDryRun() throws InterruptedException, TimeoutException {
String tableName = "test_query_job_table";

0 comments on commit ba528df

Please sign in to comment.