Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Use location in BigQueryOption as the default for query #3047

Merged
merged 6 commits into from Dec 19, 2023
Merged
8 changes: 4 additions & 4 deletions README.md
Expand Up @@ -53,20 +53,20 @@ If you are using Maven without the BOM, add this to your dependencies:
If you are using Gradle 5.x or later, add this to your dependencies:

```Groovy
implementation platform('com.google.cloud:libraries-bom:26.27.0')
implementation platform('com.google.cloud:libraries-bom:26.29.0')

implementation 'com.google.cloud:google-cloud-bigquery'
```
If you are using Gradle without BOM, add this to your dependencies:

```Groovy
implementation 'com.google.cloud:google-cloud-bigquery:2.34.2'
implementation 'com.google.cloud:google-cloud-bigquery:2.35.0'
```

If you are using SBT, add this to your dependencies:

```Scala
libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.34.2"
libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.35.0"
```
<!-- {x-version-update-end} -->

Expand Down Expand Up @@ -351,7 +351,7 @@ Java is a registered trademark of Oracle and/or its affiliates.
[kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-bigquery/java11.html
[stability-image]: https://img.shields.io/badge/stability-stable-green
[maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-bigquery.svg
[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-bigquery/2.34.2
[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-bigquery/2.35.0
[authentication]: https://github.com/googleapis/google-cloud-java#authentication
[auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes
[predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles
Expand Down
Expand Up @@ -1339,6 +1339,9 @@ public TableResult query(QueryJobConfiguration configuration, JobOption... optio
if (requestInfo.isFastQuerySupported(null)) {
String projectId = getOptions().getProjectId();
QueryRequest content = requestInfo.toPb();
if (getOptions().getLocation() != null) {
content.setLocation(getOptions().getLocation());
}
return queryRpc(projectId, content, options);
}
// Otherwise, fall back to the existing create query job logic
Expand Down Expand Up @@ -1437,12 +1440,14 @@ public TableResult query(QueryJobConfiguration configuration, JobId jobId, JobOp
String projectId =
jobId.getProject() != null ? jobId.getProject() : getOptions().getProjectId();
QueryRequest content = requestInfo.toPb();
// Be careful when setting the location in JobId, if a location is specified in the JobId,
// the job created by the query method will be in that location, even if the table to be
// Be careful when setting the location, if a location is specified in the BigQueryOption or
// JobId the job created by the query method will be in that location, even if the table to be
// queried is in a different location. This may cause the query to fail with
// "BigQueryException: Not found"
if (jobId.getLocation() != null) {
content.setLocation(jobId.getLocation());
} else if (getOptions().getLocation() != null) {
content.setLocation(getOptions().getLocation());
}

return queryRpc(projectId, content, options);
Expand Down
Expand Up @@ -1987,6 +1987,47 @@ public void testFastQueryRequestCompleted() throws InterruptedException {
QUERY_JOB_CONFIGURATION_FOR_QUERY.getDefaultDataset().getDataset(),
requestPb.getDefaultDataset().getDatasetId());
assertEquals(QUERY_JOB_CONFIGURATION_FOR_QUERY.useQueryCache(), requestPb.getUseQueryCache());
assertNull(requestPb.getLocation());

verify(bigqueryRpcMock).queryRpc(eq(PROJECT), requestPbCapture.capture());
}

@Test
public void testFastQueryRequestCompletedWithLocation() throws InterruptedException {
com.google.api.services.bigquery.model.QueryResponse queryResponsePb =
new com.google.api.services.bigquery.model.QueryResponse()
.setCacheHit(false)
.setJobComplete(true)
.setKind("bigquery#queryResponse")
.setPageToken(null)
.setRows(ImmutableList.of(TABLE_ROW))
.setSchema(TABLE_SCHEMA.toPb())
.setTotalBytesProcessed(42L)
.setTotalRows(BigInteger.valueOf(1L));

when(bigqueryRpcMock.queryRpc(eq(PROJECT), requestPbCapture.capture()))
.thenReturn(queryResponsePb);

BigQueryOptions options = createBigQueryOptionsForProjectWithLocation(PROJECT, rpcFactoryMock);
bigquery = options.getService();
TableResult result = bigquery.query(QUERY_JOB_CONFIGURATION_FOR_QUERY);
assertNull(result.getNextPage());
assertNull(result.getNextPageToken());
assertFalse(result.hasNextPage());
assertThat(result.getSchema()).isEqualTo(TABLE_SCHEMA);
assertThat(result.getTotalRows()).isEqualTo(1);
for (FieldValueList row : result.getValues()) {
assertThat(row.get(0).getBooleanValue()).isFalse();
assertThat(row.get(1).getLongValue()).isEqualTo(1);
}

QueryRequest requestPb = requestPbCapture.getValue();
assertEquals(QUERY_JOB_CONFIGURATION_FOR_QUERY.getQuery(), requestPb.getQuery());
assertEquals(
QUERY_JOB_CONFIGURATION_FOR_QUERY.getDefaultDataset().getDataset(),
requestPb.getDefaultDataset().getDatasetId());
assertEquals(QUERY_JOB_CONFIGURATION_FOR_QUERY.useQueryCache(), requestPb.getUseQueryCache());
assertEquals(LOCATION, requestPb.getLocation());

verify(bigqueryRpcMock).queryRpc(eq(PROJECT), requestPbCapture.capture());
}
Expand Down
Expand Up @@ -6227,27 +6227,80 @@ public void testAlreadyExistJobExceptionHandling() throws InterruptedException {

@Test
public void testStatelessQueries() throws InterruptedException {
// Create local BigQuery to not contaminate global test parameters.
RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create();
BigQuery bigQuery = bigqueryHelper.getOptions().getService();

// simulate setting the QUERY_PREVIEW_ENABLED environment variable
bigquery.getOptions().setQueryPreviewEnabled("TRUE");
assertNull(executeSimpleQuery().getJobId());
bigQuery.getOptions().setQueryPreviewEnabled("TRUE");
assertNull(executeSimpleQuery(bigQuery).getJobId());

// the flag should be case-insensitive
bigquery.getOptions().setQueryPreviewEnabled("tRuE");
assertNull(executeSimpleQuery().getJobId());
bigQuery.getOptions().setQueryPreviewEnabled("tRuE");
assertNull(executeSimpleQuery(bigQuery).getJobId());

// any other values won't enable optional job creation mode
bigquery.getOptions().setQueryPreviewEnabled("test_value");
assertNotNull(executeSimpleQuery().getJobId());
bigQuery.getOptions().setQueryPreviewEnabled("test_value");
assertNotNull(executeSimpleQuery(bigQuery).getJobId());

// reset the flag
bigquery.getOptions().setQueryPreviewEnabled(null);
assertNotNull(executeSimpleQuery().getJobId());
bigQuery.getOptions().setQueryPreviewEnabled(null);
assertNotNull(executeSimpleQuery(bigQuery).getJobId());
}

private TableResult executeSimpleQuery() throws InterruptedException {
private TableResult executeSimpleQuery(BigQuery bigQuery) throws InterruptedException {
String query = "SELECT 1 as one";
QueryJobConfiguration config = QueryJobConfiguration.newBuilder(query).build();
TableResult result = bigquery.query(config);
TableResult result = bigQuery.query(config);
return result;
}

@Test
public void testStatelessQueriesWithLocation() throws Exception {
// This test validates BigQueryOption location is used for stateless query by verifying that the
// stateless query fails when the BigQueryOption location does not match the dataset location.
String location = "EU";
String wrongLocation = "US";

RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create();
BigQuery bigQuery =
bigqueryHelper.getOptions().toBuilder().setLocation(location).build().getService();

Dataset dataset =
bigQuery.create(
DatasetInfo.newBuilder("locationset_" + UUID.randomUUID().toString().replace("-", "_"))
.setLocation(location)
.build());
try {
TableId tableId = TableId.of(dataset.getDatasetId().getDataset(), "sometable");
Schema schema = Schema.of(Field.of("name", LegacySQLTypeName.STRING));
TableDefinition tableDef = StandardTableDefinition.of(schema);
Table table = bigQuery.create(TableInfo.newBuilder(tableId, tableDef).build());

String query =
String.format(
"SELECT * FROM `%s.%s.%s`",
table.getTableId().getProject(),
table.getTableId().getDataset(),
table.getTableId().getTable());

// Test stateless query when BigQueryOption location matches dataset location.
bigQuery.getOptions().setQueryPreviewEnabled("TRUE");
TableResult tb = bigQuery.query(QueryJobConfiguration.of(query));
assertNull(tb.getJobId());

// Test stateless query when BigQueryOption location does not match dataset location.
try {
BigQuery bigQueryWrongLocation =
bigqueryHelper.getOptions().toBuilder().setLocation(wrongLocation).build().getService();
bigQueryWrongLocation.getOptions().setQueryPreviewEnabled("TRUE");
bigQueryWrongLocation.query(QueryJobConfiguration.of(query));
fail("querying a table with wrong location shouldn't work");
} catch (BigQueryException e) {
// Nothing to do
}
} finally {
bigQuery.delete(dataset.getDatasetId(), DatasetDeleteOption.deleteContents());
}
}
}