From 975df05b95b714c5574155d5e09860885c4b58f2 Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Wed, 17 Apr 2024 10:03:55 -0700 Subject: [PATCH] feat: Add support for ObjectMetadata (#3217) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Add support for ObjectMetadata Fixes #3216 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot --- .../bigquery/ExternalTableDefinition.java | 50 +++++++++++++++++++ .../bigquery/ExternalTableDefinitionTest.java | 3 ++ .../cloud/bigquery/it/ITBigQueryTest.java | 43 ++++++++++++++++ 3 files changed, 96 insertions(+) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java index d307b8232..9f8aa4012 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java @@ -180,6 +180,12 @@ public Builder setHivePartitioningOptions(HivePartitioningOptions hivePartitioni abstract Builder setHivePartitioningOptionsInner( HivePartitioningOptions hivePartitioningOptions); + public Builder setObjectMetadata(String objectMetadata) { + return setObjectMetadataInner(objectMetadata); + } + + abstract Builder setObjectMetadataInner(String objectMetadata); + /** Creates an {@code ExternalTableDefinition} object. */ @Override public abstract ExternalTableDefinition build(); @@ -255,6 +261,21 @@ public String getFileSetSpecType() { @Nullable public abstract ImmutableList getSourceUrisImmut(); + /** + * Returns the object metadata. + * + * @see + * ObjectMetadata + */ + @Nullable + public String getObjectMetadata() { + return getObjectMetadataInner(); + } + + @Nullable + abstract String getObjectMetadataInner(); + /** * Returns the source format, and possibly some parsing options, of the external data. Supported * formats are {@code CSV} and {@code NEWLINE_DELIMITED_JSON}. @@ -362,6 +383,10 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toExternalDataC externalConfigurationPb.setFileSetSpecType(getFileSetSpecType()); } + if (getObjectMetadata() != null) { + externalConfigurationPb.setObjectMetadata(getObjectMetadata()); + } + return externalConfigurationPb; } @@ -426,6 +451,24 @@ public static Builder newBuilder(String sourceUri, FormatOptions format) { return newBuilder().setSourceUris(ImmutableList.of(sourceUri)).setFormatOptions(format); } + /** + * Creates a builder for an ExternalTableDefinition object. + * + * @param sourceUri the fully-qualified URIs that point to your data in Google Cloud. For Google + * Cloud Bigtable URIs: Exactly one URI can be specified and it has be a fully specified and + * valid HTTPS URL for a Google Cloud Bigtable table. Size limits related to load jobs apply + * to external data sources, plus an additional limit of 10 GB maximum size across all URIs. + * @return a builder for an ExternalTableDefinition object given source URIs and format + * @see Quota + * @see + * Source Format + */ + public static Builder newBuilder(String sourceUri) { + checkArgument(!isNullOrEmpty(sourceUri), "Provided sourceUri is null or empty"); + return newBuilder().setSourceUris(ImmutableList.of(sourceUri)); + } + /** * Creates an ExternalTableDefinition object. * @@ -534,6 +577,9 @@ static ExternalTableDefinition fromPb(Table tablePb) { if (externalDataConfiguration.getFileSetSpecType() != null) { builder.setFileSetSpecType(externalDataConfiguration.getFileSetSpecType()); } + if (externalDataConfiguration.getObjectMetadata() != null) { + builder.setObjectMetadata(externalDataConfiguration.getObjectMetadata()); + } } return builder.build(); } @@ -597,6 +643,10 @@ static ExternalTableDefinition fromExternalDataConfiguration( builder.setFileSetSpecType(externalDataConfiguration.getFileSetSpecType()); } + if (externalDataConfiguration.getObjectMetadata() != null) { + builder.setObjectMetadata(externalDataConfiguration.getObjectMetadata()); + } + return builder.build(); } } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalTableDefinitionTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalTableDefinitionTest.java index 3e67ad959..ed5659891 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalTableDefinitionTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalTableDefinitionTest.java @@ -58,6 +58,7 @@ public class ExternalTableDefinitionTest { .setMode("AUTO") .setSourceUriPrefix(SOURCE_URIS.get(0)) .build(); + private static final String OBJECT_METADATA = "SIMPLE"; private static final ExternalTableDefinition EXTERNAL_TABLE_DEFINITION = ExternalTableDefinition.newBuilder(SOURCE_URIS, TABLE_SCHEMA, CSV_OPTIONS) .setFileSetSpecType("FILE_SET_SPEC_TYPE_FILE_SYSTEM_MATCH") @@ -68,6 +69,7 @@ public class ExternalTableDefinitionTest { .setMaxBadRecords(MAX_BAD_RECORDS) .setAutodetect(AUTODETECT) .setHivePartitioningOptions(HIVE_PARTITIONING_OPTIONS) + .setObjectMetadata(OBJECT_METADATA) .build(); private static final ExternalTableDefinition EXTERNAL_TABLE_DEFINITION_AVRO = @@ -167,5 +169,6 @@ private void compareExternalTableDefinition( assertEquals(expected.hashCode(), value.hashCode()); assertEquals(expected.getAutodetect(), value.getAutodetect()); assertEquals(expected.getHivePartitioningOptions(), value.getHivePartitioningOptions()); + assertEquals(expected.getObjectMetadata(), value.getObjectMetadata()); } } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 9980e69f8..9f5c6edd8 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -6579,6 +6579,49 @@ public void testExternalTableMetadataCachingNotEnable() throws InterruptedExcept assertTrue(remoteTable.delete()); } + @Test + public void testObjectTable() throws InterruptedException { + String tableName = "test_object_table"; + TableId tableId = TableId.of(DATASET, tableName); + + String sourceUri = "gs://" + BUCKET + "/" + JSON_LOAD_FILE; + ExternalTableDefinition externalTableDefinition = + ExternalTableDefinition.newBuilder(sourceUri) + .setConnectionId( + "projects/java-docs-samples-testing/locations/us/connections/DEVREL_TEST_CONNECTION") + .setObjectMetadata("SIMPLE") + .build(); + TableInfo tableInfo = TableInfo.of(tableId, externalTableDefinition); + Table createdTable = bigquery.create(tableInfo); + assertNotNull(createdTable); + assertEquals(DATASET, createdTable.getTableId().getDataset()); + assertEquals(tableName, createdTable.getTableId().getTable()); + Table remoteTable = bigquery.getTable(DATASET, tableName); + assertNotNull(remoteTable); + + try { + assertTrue(remoteTable.getDefinition() instanceof ExternalTableDefinition); + assertEquals(createdTable.getTableId(), remoteTable.getTableId()); + assertEquals( + "SIMPLE", ((ExternalTableDefinition) remoteTable.getDefinition()).getObjectMetadata()); + assertNotNull(remoteTable.getDefinition().getSchema().getFields().get("uri")); + + String query = String.format("SELECT * FROM %s.%s", DATASET, tableName); + QueryJobConfiguration config = QueryJobConfiguration.newBuilder(query).build(); + + Job remoteJob = bigquery.create(JobInfo.of(config)); + remoteJob = remoteJob.waitFor(); + assertNull(remoteJob.getStatus().getError()); + + Job queryJob = bigquery.getJob(remoteJob.getJobId()); + JobStatistics.QueryStatistics statistics = queryJob.getStatistics(); + assertNotNull(statistics); + assertThat(statistics.getTotalBytesProcessed()).isGreaterThan(0); + } finally { + assertTrue(remoteTable.delete()); + } + } + static GoogleCredentials loadCredentials(String credentialFile) { try { InputStream keyStream = new ByteArrayInputStream(credentialFile.getBytes());