Skip to content

Commit

Permalink
feat: Add support for ObjectMetadata (#3217)
Browse files Browse the repository at this point in the history
* feat: Add support for ObjectMetadata

Fixes #3216

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

---------

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
  • Loading branch information
anoopj and gcf-owl-bot[bot] committed Apr 17, 2024
1 parent d45d168 commit 975df05
Show file tree
Hide file tree
Showing 3 changed files with 96 additions and 0 deletions.
Expand Up @@ -180,6 +180,12 @@ public Builder setHivePartitioningOptions(HivePartitioningOptions hivePartitioni
abstract Builder setHivePartitioningOptionsInner(
HivePartitioningOptions hivePartitioningOptions);

public Builder setObjectMetadata(String objectMetadata) {
return setObjectMetadataInner(objectMetadata);
}

abstract Builder setObjectMetadataInner(String objectMetadata);

/** Creates an {@code ExternalTableDefinition} object. */
@Override
public abstract ExternalTableDefinition build();
Expand Down Expand Up @@ -255,6 +261,21 @@ public String getFileSetSpecType() {
@Nullable
public abstract ImmutableList<String> getSourceUrisImmut();

/**
* Returns the object metadata.
*
* @see <a
* href="https://cloud.google.com/bigquery/docs/reference/v2/tables#externalDataConfiguration">
* ObjectMetadata</a>
*/
@Nullable
public String getObjectMetadata() {
return getObjectMetadataInner();
}

@Nullable
abstract String getObjectMetadataInner();

/**
* Returns the source format, and possibly some parsing options, of the external data. Supported
* formats are {@code CSV} and {@code NEWLINE_DELIMITED_JSON}.
Expand Down Expand Up @@ -362,6 +383,10 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toExternalDataC
externalConfigurationPb.setFileSetSpecType(getFileSetSpecType());
}

if (getObjectMetadata() != null) {
externalConfigurationPb.setObjectMetadata(getObjectMetadata());
}

return externalConfigurationPb;
}

Expand Down Expand Up @@ -426,6 +451,24 @@ public static Builder newBuilder(String sourceUri, FormatOptions format) {
return newBuilder().setSourceUris(ImmutableList.of(sourceUri)).setFormatOptions(format);
}

/**
* Creates a builder for an ExternalTableDefinition object.
*
* @param sourceUri the fully-qualified URIs that point to your data in Google Cloud. For Google
* Cloud Bigtable URIs: Exactly one URI can be specified and it has be a fully specified and
* valid HTTPS URL for a Google Cloud Bigtable table. Size limits related to load jobs apply
* to external data sources, plus an additional limit of 10 GB maximum size across all URIs.
* @return a builder for an ExternalTableDefinition object given source URIs and format
* @see <a href="https://cloud.google.com/bigquery/loading-data-into-bigquery#quota">Quota</a>
* @see <a
* href="https://cloud.google.com/bigquery/docs/reference/v2/tables#externalDataConfiguration.sourceFormat">
* Source Format</a>
*/
public static Builder newBuilder(String sourceUri) {
checkArgument(!isNullOrEmpty(sourceUri), "Provided sourceUri is null or empty");
return newBuilder().setSourceUris(ImmutableList.of(sourceUri));
}

/**
* Creates an ExternalTableDefinition object.
*
Expand Down Expand Up @@ -534,6 +577,9 @@ static ExternalTableDefinition fromPb(Table tablePb) {
if (externalDataConfiguration.getFileSetSpecType() != null) {
builder.setFileSetSpecType(externalDataConfiguration.getFileSetSpecType());
}
if (externalDataConfiguration.getObjectMetadata() != null) {
builder.setObjectMetadata(externalDataConfiguration.getObjectMetadata());
}
}
return builder.build();
}
Expand Down Expand Up @@ -597,6 +643,10 @@ static ExternalTableDefinition fromExternalDataConfiguration(
builder.setFileSetSpecType(externalDataConfiguration.getFileSetSpecType());
}

if (externalDataConfiguration.getObjectMetadata() != null) {
builder.setObjectMetadata(externalDataConfiguration.getObjectMetadata());
}

return builder.build();
}
}
Expand Up @@ -58,6 +58,7 @@ public class ExternalTableDefinitionTest {
.setMode("AUTO")
.setSourceUriPrefix(SOURCE_URIS.get(0))
.build();
private static final String OBJECT_METADATA = "SIMPLE";
private static final ExternalTableDefinition EXTERNAL_TABLE_DEFINITION =
ExternalTableDefinition.newBuilder(SOURCE_URIS, TABLE_SCHEMA, CSV_OPTIONS)
.setFileSetSpecType("FILE_SET_SPEC_TYPE_FILE_SYSTEM_MATCH")
Expand All @@ -68,6 +69,7 @@ public class ExternalTableDefinitionTest {
.setMaxBadRecords(MAX_BAD_RECORDS)
.setAutodetect(AUTODETECT)
.setHivePartitioningOptions(HIVE_PARTITIONING_OPTIONS)
.setObjectMetadata(OBJECT_METADATA)
.build();

private static final ExternalTableDefinition EXTERNAL_TABLE_DEFINITION_AVRO =
Expand Down Expand Up @@ -167,5 +169,6 @@ private void compareExternalTableDefinition(
assertEquals(expected.hashCode(), value.hashCode());
assertEquals(expected.getAutodetect(), value.getAutodetect());
assertEquals(expected.getHivePartitioningOptions(), value.getHivePartitioningOptions());
assertEquals(expected.getObjectMetadata(), value.getObjectMetadata());
}
}
Expand Up @@ -6579,6 +6579,49 @@ public void testExternalTableMetadataCachingNotEnable() throws InterruptedExcept
assertTrue(remoteTable.delete());
}

@Test
public void testObjectTable() throws InterruptedException {
String tableName = "test_object_table";
TableId tableId = TableId.of(DATASET, tableName);

String sourceUri = "gs://" + BUCKET + "/" + JSON_LOAD_FILE;
ExternalTableDefinition externalTableDefinition =
ExternalTableDefinition.newBuilder(sourceUri)
.setConnectionId(
"projects/java-docs-samples-testing/locations/us/connections/DEVREL_TEST_CONNECTION")
.setObjectMetadata("SIMPLE")
.build();
TableInfo tableInfo = TableInfo.of(tableId, externalTableDefinition);
Table createdTable = bigquery.create(tableInfo);
assertNotNull(createdTable);
assertEquals(DATASET, createdTable.getTableId().getDataset());
assertEquals(tableName, createdTable.getTableId().getTable());
Table remoteTable = bigquery.getTable(DATASET, tableName);
assertNotNull(remoteTable);

try {
assertTrue(remoteTable.getDefinition() instanceof ExternalTableDefinition);
assertEquals(createdTable.getTableId(), remoteTable.getTableId());
assertEquals(
"SIMPLE", ((ExternalTableDefinition) remoteTable.getDefinition()).getObjectMetadata());
assertNotNull(remoteTable.getDefinition().getSchema().getFields().get("uri"));

String query = String.format("SELECT * FROM %s.%s", DATASET, tableName);
QueryJobConfiguration config = QueryJobConfiguration.newBuilder(query).build();

Job remoteJob = bigquery.create(JobInfo.of(config));
remoteJob = remoteJob.waitFor();
assertNull(remoteJob.getStatus().getError());

Job queryJob = bigquery.getJob(remoteJob.getJobId());
JobStatistics.QueryStatistics statistics = queryJob.getStatistics();
assertNotNull(statistics);
assertThat(statistics.getTotalBytesProcessed()).isGreaterThan(0);
} finally {
assertTrue(remoteTable.delete());
}
}

static GoogleCredentials loadCredentials(String credentialFile) {
try {
InputStream keyStream = new ByteArrayInputStream(credentialFile.getBytes());
Expand Down

0 comments on commit 975df05

Please sign in to comment.