From 3895bd94b283b6ff731cfa94426ea0691e0d54c4 Mon Sep 17 00:00:00 2001 From: emkornfield Date: Mon, 25 Sep 2023 04:33:52 -0700 Subject: [PATCH] feat: Add support for FileSetSpec (#2888) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Add support for FileSetSpec * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot --- .../bigquery/ExternalTableDefinition.java | 30 +++++++++++++++++++ .../cloud/bigquery/LoadJobConfiguration.java | 26 ++++++++++++++++ .../bigquery/ExternalTableDefinitionTest.java | 2 ++ .../bigquery/LoadJobConfigurationTest.java | 2 ++ 4 files changed, 60 insertions(+) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java index 18c78e750..d307b8232 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java @@ -79,8 +79,20 @@ public Builder setSourceUris(List sourceUris) { return setSourceUrisImmut(ImmutableList.copyOf(sourceUris)); } + abstract Builder setFileSetSpecTypeInner(String spec); + abstract Builder setSourceUrisImmut(ImmutableList sourceUris); + /** + * Defines how to interpret files denoted by URIs. By default the files are assumed to be data + * files (this can be specified explicitly via FILE_SET_SPEC_TYPE_FILE_SYSTEM_MATCH). A second + * option is "FILE_SET_SPEC_TYPE_NEW_LINE_DELIMITED_MANIFEST" which interprets each file as a + * manifest file, where each line is a reference to a file. + */ + public Builder setFileSetSpecType(String fileSetSpecType) { + return setFileSetSpecTypeInner(fileSetSpecType); + } + /** * Sets the source format, and possibly some parsing options, of the external data. Supported * formats are {@code CSV} and {@code NEWLINE_DELIMITED_JSON}. @@ -232,6 +244,14 @@ public List getSourceUris() { return getSourceUrisImmut(); } + @Nullable + public String getFileSetSpecType() { + return getFileSetSpecTypeInner(); + } + + @Nullable + abstract String getFileSetSpecTypeInner(); + @Nullable public abstract ImmutableList getSourceUrisImmut(); @@ -338,6 +358,10 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toExternalDataC if (getHivePartitioningOptions() != null) { externalConfigurationPb.setHivePartitioningOptions(getHivePartitioningOptions().toPb()); } + if (getFileSetSpecType() != null) { + externalConfigurationPb.setFileSetSpecType(getFileSetSpecType()); + } + return externalConfigurationPb; } @@ -507,6 +531,9 @@ static ExternalTableDefinition fromPb(Table tablePb) { if (externalDataConfiguration.getReferenceFileSchemaUri() != null) { builder.setReferenceFileSchemaUri(externalDataConfiguration.getReferenceFileSchemaUri()); } + if (externalDataConfiguration.getFileSetSpecType() != null) { + builder.setFileSetSpecType(externalDataConfiguration.getFileSetSpecType()); + } } return builder.build(); } @@ -566,6 +593,9 @@ static ExternalTableDefinition fromExternalDataConfiguration( builder.setHivePartitioningOptions( HivePartitioningOptions.fromPb(externalDataConfiguration.getHivePartitioningOptions())); } + if (externalDataConfiguration.getFileSetSpecType() != null) { + builder.setFileSetSpecType(externalDataConfiguration.getFileSetSpecType()); + } return builder.build(); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java index 321d542a6..fefff3409 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java @@ -38,6 +38,7 @@ public final class LoadJobConfiguration extends JobConfiguration implements Load private static final long serialVersionUID = -2673554846792429829L; private final List sourceUris; + private final String fileSetSpecType; private final TableId destinationTable; private final List decimalTargetTypes; private final EncryptionConfiguration destinationEncryptionConfiguration; @@ -67,6 +68,7 @@ public static final class Builder extends JobConfiguration.Builder sourceUris; + private String fileSetSpecType; private TableId destinationTable; private List decimalTargetTypes; private EncryptionConfiguration destinationEncryptionConfiguration; @@ -107,6 +109,7 @@ private Builder(LoadJobConfiguration loadConfiguration) { this.schema = loadConfiguration.schema; this.ignoreUnknownValues = loadConfiguration.ignoreUnknownValues; this.sourceUris = loadConfiguration.sourceUris; + this.fileSetSpecType = loadConfiguration.fileSetSpecType; this.schemaUpdateOptions = loadConfiguration.schemaUpdateOptions; this.autodetect = loadConfiguration.autodetect; this.destinationEncryptionConfiguration = @@ -175,6 +178,9 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur if (loadConfigurationPb.getSourceUris() != null) { this.sourceUris = ImmutableList.copyOf(configurationPb.getLoad().getSourceUris()); } + if (loadConfigurationPb.getFileSetSpecType() != null) { + this.fileSetSpecType = loadConfigurationPb.getFileSetSpecType(); + } if (loadConfigurationPb.getSchemaUpdateOptions() != null) { ImmutableList.Builder schemaUpdateOptionsBuilder = new ImmutableList.Builder<>(); @@ -306,6 +312,17 @@ public Builder setSourceUris(List sourceUris) { return this; } + /** + * Defines how to interpret files denoted by URIs. By default the files are assumed to be data + * files (this can be specified explicitly via FILE_SET_SPEC_TYPE_FILE_SYSTEM_MATCH). A second + * option is "FILE_SET_SPEC_TYPE_NEW_LINE_DELIMITED_MANIFEST" which interprets each file as a + * manifest file, where each line is a reference to a file. + */ + public Builder setFileSetSpecType(String fileSetSpecType) { + this.fileSetSpecType = fileSetSpecType; + return this; + } + /** * Defines the list of possible SQL data types to which the source decimal values are converted. * This list and the precision and the scale parameters of the decimal field determine the @@ -403,6 +420,7 @@ public LoadJobConfiguration build() { private LoadJobConfiguration(Builder builder) { super(builder); this.sourceUris = builder.sourceUris; + this.fileSetSpecType = builder.fileSetSpecType; this.destinationTable = builder.destinationTable; this.decimalTargetTypes = builder.decimalTargetTypes; this.createDisposition = builder.createDisposition; @@ -497,6 +515,10 @@ public List getSourceUris() { return sourceUris; } + public String getFileSetSpecType() { + return fileSetSpecType; + } + public List getDecimalTargetTypes() { return decimalTargetTypes; } @@ -575,6 +597,7 @@ ToStringHelper toStringHelper() { .add("schema", schema) .add("ignoreUnknownValue", ignoreUnknownValues) .add("sourceUris", sourceUris) + .add("fileSetSpecType", fileSetSpecType) .add("schemaUpdateOptions", schemaUpdateOptions) .add("autodetect", autodetect) .add("timePartitioning", timePartitioning) @@ -655,6 +678,9 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() { if (sourceUris != null) { loadConfigurationPb.setSourceUris(ImmutableList.copyOf(sourceUris)); } + if (fileSetSpecType != null) { + loadConfigurationPb.setFileSetSpecType(fileSetSpecType); + } if (decimalTargetTypes != null) { loadConfigurationPb.setDecimalTargetTypes(ImmutableList.copyOf(decimalTargetTypes)); } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalTableDefinitionTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalTableDefinitionTest.java index 4a7409162..3e67ad959 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalTableDefinitionTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalTableDefinitionTest.java @@ -60,6 +60,7 @@ public class ExternalTableDefinitionTest { .build(); private static final ExternalTableDefinition EXTERNAL_TABLE_DEFINITION = ExternalTableDefinition.newBuilder(SOURCE_URIS, TABLE_SCHEMA, CSV_OPTIONS) + .setFileSetSpecType("FILE_SET_SPEC_TYPE_FILE_SYSTEM_MATCH") .setDecimalTargetTypes(DECIMAL_TARGET_TYPES) .setCompression(COMPRESSION) .setConnectionId(CONNECTION_ID) @@ -154,6 +155,7 @@ public void testToAndFromPbParquet() { private void compareExternalTableDefinition( ExternalTableDefinition expected, ExternalTableDefinition value) { assertEquals(expected, value); + assertEquals(expected.getFileSetSpecType(), value.getFileSetSpecType()); assertEquals(expected.getDecimalTargetTypes(), value.getDecimalTargetTypes()); assertEquals(expected.getCompression(), value.getCompression()); assertEquals(expected.getConnectionId(), value.getConnectionId()); diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java index 341965fb8..563a3f34a 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java @@ -91,6 +91,7 @@ public class LoadJobConfigurationTest { .setCreateDisposition(CREATE_DISPOSITION) .setWriteDisposition(WRITE_DISPOSITION) .setFormatOptions(CSV_OPTIONS) + .setFileSetSpecType("FILE_SET_SPEC_TYPE_FILE_SYSTEM_MATCH") .setIgnoreUnknownValues(IGNORE_UNKNOWN_VALUES) .setMaxBadRecords(MAX_BAD_RECORDS) .setSchema(TABLE_SCHEMA) @@ -240,6 +241,7 @@ private void compareLoadJobConfiguration( LoadJobConfiguration expected, LoadJobConfiguration value) { assertEquals(expected, value); assertEquals(expected.hashCode(), value.hashCode()); + assertEquals(expected.getFileSetSpecType(), value.getFileSetSpecType()); assertEquals(expected.toString(), value.toString()); assertEquals(expected.getDestinationTable(), value.getDestinationTable()); assertEquals(expected.getDecimalTargetTypes(), value.getDecimalTargetTypes());