Skip to content

Commit

Permalink
feat: Partial Projection of Table Metadata (#2756)
Browse files Browse the repository at this point in the history 
Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly:
- [ ] Make sure to open an issue as a [bug/issue](https://togithub.com/googleapis/java-bigquery/issues/new/choose) before writing your code!  That way we can discuss the change, evaluate designs, and agree on the general idea
- [ ] Ensure the tests and linter pass
- [ ] Code coverage does not decrease (if any source code was changed)
- [ ] Appropriate docs were updated (if necessary)

Fixes #<issue_number_goes_here> ☕️

If you write sample code, please follow the [samples format](
https://togithub.com/GoogleCloudPlatform/java-docs-samples/blob/main/SAMPLE_FORMAT.md).
  • Loading branch information
@farhan0102
farhan0102 committed Jun 19, 2023
1 parent afb571c commit 9207743
Show file tree
Hide file tree
Showing 5 changed files with 160 additions and 4 deletions.
19 changes: 19 additions & 0 deletions google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java
View file
Expand Up @@ -121,6 +121,20 @@ public String getSelector() {
}
}

/**
* Metadata of a BigQuery Table.
*
* @see <a href=
* "https://cloud.google.com/bigquery/docs/reference/rest/v2/tables/get#tablemetadataview">Table
* Resource</a>
*/
enum TableMetadataView {
BASIC,
FULL,
STORAGE_STATS,
TABLE_METADATA_VIEW_UNSPECIFIED;
}

/**
* Fields of a BigQuery Model resource.
*
Expand Down Expand Up @@ -384,6 +398,11 @@ public static TableOption fields(TableField... fields) {
public static TableOption autodetectSchema(boolean autodetect) {
return new TableOption(BigQueryRpc.Option.AUTODETECT_SCHEMA, autodetect);
}

/** Returns an option to specify the metadata of the table. */
public static TableOption tableMetadataView(TableMetadataView tableMetadataView) {
return new TableOption(BigQueryRpc.Option.TABLE_METADATA_VIEW, tableMetadataView);
}
}

/* Class for specifying IAM options. */
Expand Down
4 changes: 2 additions & 2 deletions google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Table.java
View file
Expand Up @@ -49,10 +49,10 @@ public static class Builder extends TableInfo.Builder {
private final BigQuery bigquery;
private final TableInfo.BuilderImpl infoBuilder;

Builder(BigQuery bigquery, TableId tableId, TableDefinition defintion) {
Builder(BigQuery bigquery, TableId tableId, TableDefinition definition) {
this.bigquery = bigquery;
this.infoBuilder = new TableInfo.BuilderImpl();
this.infoBuilder.setTableId(tableId).setDefinition(defintion);
this.infoBuilder.setTableId(tableId).setDefinition(definition);
}

Builder(Table table) {
Expand Down
5 changes: 3 additions & 2 deletions google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java
View file
Expand Up @@ -40,7 +40,7 @@
@InternalExtensionOnly
public interface BigQueryRpc extends ServiceRpc {

// These options are part of the Google Cloud BigQuery query parameters
// These options are part of the Google Cloud BigQuery query parameters.
enum Option {
FIELDS("fields"),
DELETE_CONTENTS("deleteContents"),
Expand All @@ -56,7 +56,8 @@ enum Option {
START_INDEX("startIndex"),
STATE_FILTER("stateFilter"),
TIMEOUT("timeoutMs"),
REQUESTED_POLICY_VERSION("requestedPolicyVersion");
REQUESTED_POLICY_VERSION("requestedPolicyVersion"),
TABLE_METADATA_VIEW("view");

private final String value;

Expand Down
8 changes: 8 additions & 0 deletions google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java
View file
Expand Up @@ -294,6 +294,7 @@ public Table getTable(
.get(projectId, datasetId, tableId)
.setPrettyPrint(false)
.setFields(Option.FIELDS.getString(options))
.setView(getTableMetadataOption(options))
.execute();
} catch (IOException ex) {
BigQueryException serviceException = translate(ex);
Expand All @@ -304,6 +305,13 @@ public Table getTable(
}
}

private String getTableMetadataOption(Map<Option, ?> options) {
if (options.containsKey(Option.TABLE_METADATA_VIEW)) {
return options.get(Option.TABLE_METADATA_VIEW).toString();
}
return "STORAGE_STATS";
}

@Override
public Tuple<String, Iterable<Table>> listTables(
String projectId, String datasetId, Map<Option, ?> options) {
Expand Down
128 changes: 128 additions & 0 deletions google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java
View file
Expand Up @@ -49,6 +49,7 @@
import com.google.cloud.bigquery.BigQuery.JobListOption;
import com.google.cloud.bigquery.BigQuery.JobOption;
import com.google.cloud.bigquery.BigQuery.TableField;
import com.google.cloud.bigquery.BigQuery.TableMetadataView;
import com.google.cloud.bigquery.BigQuery.TableOption;
import com.google.cloud.bigquery.BigQueryDryRunResult;
import com.google.cloud.bigquery.BigQueryError;
Expand Down Expand Up @@ -1549,6 +1550,133 @@ public void testCreateAndGetTable() {
assertTrue(remoteTable.delete());
}

@Test
public void testCreateAndGetTableWithBasicTableMetadataView() {
String tableName = "test_create_and_get_table_with_basic_metadata_view";
TableId tableId = TableId.of(DATASET, tableName);
TimePartitioning partitioning = TimePartitioning.of(Type.DAY);
Clustering clustering =
Clustering.newBuilder().setFields(ImmutableList.of(STRING_FIELD_SCHEMA.getName())).build();
StandardTableDefinition tableDefinition =
StandardTableDefinition.newBuilder()
.setSchema(TABLE_SCHEMA)
.setTimePartitioning(partitioning)
.setClustering(clustering)
.build();
Table createdTable = bigquery.create(TableInfo.of(tableId, tableDefinition));
assertNotNull(createdTable);
assertEquals(DATASET, createdTable.getTableId().getDataset());
assertEquals(tableName, createdTable.getTableId().getTable());
TableOption tableOption = BigQuery.TableOption.tableMetadataView(TableMetadataView.BASIC);
Table remoteTable = bigquery.getTable(DATASET, tableName, tableOption);
assertNotNull(remoteTable);
assertTrue(remoteTable.getDefinition() instanceof StandardTableDefinition);
assertEquals(createdTable.getTableId(), remoteTable.getTableId());
assertEquals(TableDefinition.Type.TABLE, remoteTable.getDefinition().getType());
assertEquals(TABLE_SCHEMA, remoteTable.getDefinition().getSchema());
// Next four values are considered transient fields that should not be calculated
assertNull(remoteTable.getLastModifiedTime());
assertNull(remoteTable.<StandardTableDefinition>getDefinition().getNumBytes());
assertNull(remoteTable.<StandardTableDefinition>getDefinition().getNumLongTermBytes());
assertNull(remoteTable.<StandardTableDefinition>getDefinition().getNumRows());
assertTrue(remoteTable.delete());
}

@Test
public void testCreateAndGetTableWithFullTableMetadataView() {
String tableName = "test_create_and_get_table_with_full_metadata_view";
TableId tableId = TableId.of(DATASET, tableName);
TimePartitioning partitioning = TimePartitioning.of(Type.DAY);
Clustering clustering =
Clustering.newBuilder().setFields(ImmutableList.of(STRING_FIELD_SCHEMA.getName())).build();
StandardTableDefinition tableDefinition =
StandardTableDefinition.newBuilder()
.setSchema(TABLE_SCHEMA)
.setTimePartitioning(partitioning)
.setClustering(clustering)
.build();
Table createdTable = bigquery.create(TableInfo.of(tableId, tableDefinition));
assertNotNull(createdTable);
assertEquals(DATASET, createdTable.getTableId().getDataset());
assertEquals(tableName, createdTable.getTableId().getTable());
TableOption tableOption = BigQuery.TableOption.tableMetadataView(TableMetadataView.FULL);
Table remoteTable = bigquery.getTable(DATASET, tableName, tableOption);
assertNotNull(remoteTable);
assertTrue(remoteTable.getDefinition() instanceof StandardTableDefinition);
assertEquals(createdTable.getTableId(), remoteTable.getTableId());
assertEquals(TableDefinition.Type.TABLE, remoteTable.getDefinition().getType());
assertEquals(TABLE_SCHEMA, remoteTable.getDefinition().getSchema());
assertNotNull(remoteTable.getLastModifiedTime());
assertNotNull(remoteTable.<StandardTableDefinition>getDefinition().getNumBytes());
assertNotNull(remoteTable.<StandardTableDefinition>getDefinition().getNumLongTermBytes());
assertNotNull(remoteTable.<StandardTableDefinition>getDefinition().getNumRows());
assertTrue(remoteTable.delete());
}

@Test
public void testCreateAndGetTableWithStorageStatsTableMetadataView() {
String tableName = "test_create_and_get_table_with_storage_stats_metadata_view";
TableId tableId = TableId.of(DATASET, tableName);
TimePartitioning partitioning = TimePartitioning.of(Type.DAY);
Clustering clustering =
Clustering.newBuilder().setFields(ImmutableList.of(STRING_FIELD_SCHEMA.getName())).build();
StandardTableDefinition tableDefinition =
StandardTableDefinition.newBuilder()
.setSchema(TABLE_SCHEMA)
.setTimePartitioning(partitioning)
.setClustering(clustering)
.build();
Table createdTable = bigquery.create(TableInfo.of(tableId, tableDefinition));
assertNotNull(createdTable);
assertEquals(DATASET, createdTable.getTableId().getDataset());
assertEquals(tableName, createdTable.getTableId().getTable());
TableOption tableOption =
BigQuery.TableOption.tableMetadataView(TableMetadataView.STORAGE_STATS);
Table remoteTable = bigquery.getTable(DATASET, tableName, tableOption);
assertNotNull(remoteTable);
assertTrue(remoteTable.getDefinition() instanceof StandardTableDefinition);
assertEquals(createdTable.getTableId(), remoteTable.getTableId());
assertEquals(TableDefinition.Type.TABLE, remoteTable.getDefinition().getType());
assertEquals(TABLE_SCHEMA, remoteTable.getDefinition().getSchema());
assertNotNull(remoteTable.getLastModifiedTime());
assertNotNull(remoteTable.<StandardTableDefinition>getDefinition().getNumBytes());
assertNotNull(remoteTable.<StandardTableDefinition>getDefinition().getNumLongTermBytes());
assertNotNull(remoteTable.<StandardTableDefinition>getDefinition().getNumRows());
assertTrue(remoteTable.delete());
}

@Test
public void testCreateAndGetTableWithUnspecifiedTableMetadataView() {
String tableName = "test_create_and_get_table_with_unspecified_metadata_view";
TableId tableId = TableId.of(DATASET, tableName);
TimePartitioning partitioning = TimePartitioning.of(Type.DAY);
Clustering clustering =
Clustering.newBuilder().setFields(ImmutableList.of(STRING_FIELD_SCHEMA.getName())).build();
StandardTableDefinition tableDefinition =
StandardTableDefinition.newBuilder()
.setSchema(TABLE_SCHEMA)
.setTimePartitioning(partitioning)
.setClustering(clustering)
.build();
Table createdTable = bigquery.create(TableInfo.of(tableId, tableDefinition));
assertNotNull(createdTable);
assertEquals(DATASET, createdTable.getTableId().getDataset());
assertEquals(tableName, createdTable.getTableId().getTable());
TableOption tableOption =
BigQuery.TableOption.tableMetadataView(TableMetadataView.TABLE_METADATA_VIEW_UNSPECIFIED);
Table remoteTable = bigquery.getTable(DATASET, tableName, tableOption);
assertNotNull(remoteTable);
assertTrue(remoteTable.getDefinition() instanceof StandardTableDefinition);
assertEquals(createdTable.getTableId(), remoteTable.getTableId());
assertEquals(TableDefinition.Type.TABLE, remoteTable.getDefinition().getType());
assertEquals(TABLE_SCHEMA, remoteTable.getDefinition().getSchema());
assertNotNull(remoteTable.getLastModifiedTime());
assertNotNull(remoteTable.<StandardTableDefinition>getDefinition().getNumBytes());
assertNotNull(remoteTable.<StandardTableDefinition>getDefinition().getNumLongTermBytes());
assertNotNull(remoteTable.<StandardTableDefinition>getDefinition().getNumRows());
assertTrue(remoteTable.delete());
}

@Test
public void testCreateAndGetTableWithSelectedField() {
String tableName = "test_create_and_get_selected_fields_table";
Expand Down

0 comments on commit 9207743

Please sign in to comment.