From dd1d01d7054db057bf0871f69d6799544dafe3ef Mon Sep 17 00:00:00 2001 From: stephwang Date: Tue, 15 Jun 2021 16:16:35 -0400 Subject: [PATCH 001/277] feat: add QueryConnection interface Update QueryConnection; add BigQueryResultSet and BigQuerySQLException lint bigquerystorage client read api spec bigquerystorage client read api spec Add ReadClientConnectionConfiguration class remove extraneous configs since reading from anon query result table is free using the read client --- .../cloud/bigquery/BigQueryResultSet.java | 32 ++++ .../cloud/bigquery/BigQuerySQLException.java | 80 +++++++++ .../cloud/bigquery/QueryConnection.java | 170 ++++++++++++++++++ .../ReadClientConnectionConfiguration.java | 45 +++++ 4 files changed, 327 insertions(+) create mode 100644 google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java create mode 100644 google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuerySQLException.java create mode 100644 google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryConnection.java create mode 100644 google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ReadClientConnectionConfiguration.java diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java new file mode 100644 index 000000000..0edd0fd25 --- /dev/null +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java @@ -0,0 +1,32 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +public interface BigQueryResultSet { + + /** Returns the schema of the results. Null if the schema is not supplied. */ + Schema getSchema(); + + /** + * Returns the total number of rows in the complete result set, which can be more than the number + * of rows in the first page of results. + */ + long getTotalRows(); + + /** Returns the next page of result set. Null if there is no more pages left. */ + BigQueryResultSet getNextPage(); +} diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuerySQLException.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuerySQLException.java new file mode 100644 index 000000000..08c6aff54 --- /dev/null +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuerySQLException.java @@ -0,0 +1,80 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import java.sql.SQLException; +import java.util.List; + +/** + * BigQuery service exception. + * + * @see Google Cloud BigQuery + * error codes + */ +public final class BigQuerySQLException extends SQLException { + + private static final long serialVersionUID = -5006625989225438209L; + private final List errors; + + public BigQuerySQLException() { + this.errors = null; + } + + public BigQuerySQLException(List errors) { + this.errors = errors; + } + + public BigQuerySQLException(List errors, Throwable cause) { + super(cause != null ? cause.toString() : null); + this.errors = errors; + } + + public BigQuerySQLException(String reason, List errors) { + super(reason); + this.errors = errors; + } + + public BigQuerySQLException(String reason, Throwable cause, List errors) { + super(reason, cause); + this.errors = errors; + } + + public BigQuerySQLException(String reason, String sqlState, List errors) { + super(reason, sqlState); + this.errors = errors; + } + + public BigQuerySQLException( + String reason, String sqlState, int errorCode, List errors) { + super(reason, sqlState, errorCode); + this.errors = errors; + } + + public BigQuerySQLException( + String reason, String sqlState, int errorCode, Throwable cause, List errors) { + super(reason, sqlState, errorCode, cause); + this.errors = errors; + } + + /** + * Returns a list of {@link BigQueryError}s that caused this exception. Returns {@code null} if + * none exists. + */ + public List getErrors() { + return errors; + } +} diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryConnection.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryConnection.java new file mode 100644 index 000000000..d3cb6f81b --- /dev/null +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryConnection.java @@ -0,0 +1,170 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import com.google.api.services.bigquery.model.QueryParameter; +import java.util.List; +import java.util.Map; + +/** + * A Connection is a session between a Java application and BigQuery. SQL statements are executed + * and results are returned within the context of a connection. + */ +public interface QueryConnection { + + /** + * Sets how long to wait for the query to complete, in milliseconds, before the request times out + * and returns. Note that this is only a timeout for the request, not the query. If the query + * takes longer to run than the timeout value, the call returns without any results and with the + * 'jobComplete' flag set to false. You can call GetQueryResults() to wait for the query to + * complete and read the results. The default value is 10000 milliseconds (10 seconds). + * + * @param timeoutMs or {@code null} for none + */ + void setSynchronousResponseTimeoutMs(Long timeoutMs) throws BigQuerySQLException; + + /** Returns the synchronous response timeoutMs associated with this query */ + Long getSynchronousResponseTimeoutMs() throws BigQuerySQLException; + + /** + * Sets a connection-level property to customize query behavior. Under JDBC, these correspond + * directly to connection properties passed to the DriverManager. + * + * @param connectionProperties connectionProperties or {@code null} for none + */ + void setConnectionProperties(List connectionProperties) + throws BigQuerySQLException; + + /** Returns the connection properties for connection string with this query */ + List getConnectionProperties() throws BigQuerySQLException; + + /** + * Sets the default dataset. This dataset is used for all unqualified table names used in the + * query. + */ + void setDefaultDataset() throws BigQuerySQLException; + + /** Returns the default dataset */ + DatasetId getDefaultDataset() throws BigQuerySQLException; + + /** + * Sets the labels associated with this query. You can use these to organize and group your + * queries. Label keys and values can be no longer than 63 characters, can only contain lowercase + * letters, numeric characters, underscores and dashes. International characters are allowed. + * Label values are optional. Label keys must start with a letter and each label in the list must + * have a different key. + * + * @param labels labels or {@code null} for none + */ + void setLabels(Map labels) throws BigQuerySQLException; + + /** Returns the labels associated with this query */ + Map getLabels() throws BigQuerySQLException; + + /** Clear the labels associated with this query */ + void cleaLabels() throws BigQuerySQLException; + + /** + * Limits the bytes billed for this job. Queries that will have bytes billed beyond this limit + * will fail (without incurring a charge). If unspecified, this will be set to your project + * default. + * + * @param maximumBytesBilled maximum bytes billed for this job + */ + void setMaximumBytesBilled(Long maximumBytesBilled) throws BigQuerySQLException; + + /** Returns the limits the bytes billed for this job */ + Long getMaximumBytesBilled() throws BigQuerySQLException; + + /** + * Sets the maximum number of rows of data to return per page of results. Setting this flag to a + * small value such as 1000 and then paging through results might improve reliability when the + * query result set is large. In addition to this limit, responses are also limited to 10 MB. By + * default, there is no maximum row count, and only the byte limit applies. + * + * @param maxResults maxResults or {@code null} for none + */ + void setMaxResults(Long maxResults) throws BigQuerySQLException; + + /** Returns the maximum number of rows of data */ + Long getMaxResults() throws BigQuerySQLException; + + /** Returns query parameters for standard SQL queries */ + List getQueryParameters() throws BigQuerySQLException; + + /** + * Sets whether to look for the result in the query cache. The query cache is a best-effort cache + * that will be flushed whenever tables in the query are modified. Moreover, the query cache is + * only available when {@link QueryJobConfiguration.Builder#setDestinationTable(TableId)} is not + * set. + * + * @see Query Caching + */ + void setUseQueryCache(Boolean useQueryCache) throws BigQuerySQLException; + + /** Returns whether to look for the result in the query cache */ + Boolean getUseQueryCache() throws BigQuerySQLException; + + /** + * Execute a query dry run that does not return any BigQueryResultSet + * + * @param sql typically a static SQL SELECT statement + * @exception BigQuerySQLException if a database access error occurs + */ + void dryRun(String sql) throws BigQuerySQLException; + + /** + * Execute a SQL statement that returns a single BigQueryResultSet + * + * @param sql typically a static SQL SELECT statement + * @return a BigQueryResultSet that contains the data produced by the query + * @exception BigQuerySQLException if a database access error occurs + */ + BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException; + + /** + * Execute a SQL statement with query parameters that returns a single BigQueryResultSet + * + * @param sql typically a static SQL SELECT statement + * @param parameters named or positional parameters + * @return a BigQueryResultSet that contains the data produced by the query + * @exception BigQuerySQLException if a database access error occurs + */ + BigQueryResultSet executeSelect(String sql, List parameters) + throws BigQuerySQLException; + + /** + * Sets the values necessary to determine whether table result will be read using the BigQuery + * Storage client Read API. The BigQuery Storage client Read API will be used to read the query + * result when the totalToFirstPageSizeRatio (default 3) and minimumTableSize (default 100MB) + * conditions set are met. A ReadSession will be created using Apache Avro data format for + * serialization. + * + *

It also sets the maximum number of table rows allowed in buffer before streaming them to the + * BigQueryResultSet. + * + * @param readClientConnectionConfiguration or {@code null} for none + * @exception BigQueryException if an error occurs in setting these values + */ + void setReadClientConnectionConfiguration( + ReadClientConnectionConfiguration readClientConnectionConfiguration) throws BigQueryException; +} + +class Parameter { + String name; + QueryParameterValue value; +} diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ReadClientConnectionConfiguration.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ReadClientConnectionConfiguration.java new file mode 100644 index 000000000..8503e642a --- /dev/null +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ReadClientConnectionConfiguration.java @@ -0,0 +1,45 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import com.google.auto.value.AutoValue; +import java.io.Serializable; + +/** Represents BigQueryStorage Read client connection information. */ +@AutoValue +public abstract class ReadClientConnectionConfiguration implements Serializable { + + @AutoValue.Builder + public abstract static class Builder { + + /* Sets the buffer size during streaming from the BigQueryStorage Read client */ + public abstract Builder setBufferSize(Long bufferSize); + + /** Creates a {@code ReadClientConnectionConfiguration} object. */ + public abstract ReadClientConnectionConfiguration build(); + } + + /** Returns the bufferSize in this configuration. */ + public abstract Long getBufferSize(); + + public abstract Builder toBuilder(); + + /** Returns a builder for a {@code ReadClientConnectionConfiguration} object. */ + public static Builder newBuilder() { + return new AutoValue_ReadClientConnectionConfiguration.Builder(); + } +} From 6d4511ab89e8a251f2a898ea98ce2c0f51a76512 Mon Sep 17 00:00:00 2001 From: stephwang Date: Tue, 20 Jul 2021 11:35:32 -0400 Subject: [PATCH 002/277] updates per feedback from Simba fix linkage monitor error update Javadoc some clean up some clean up adding impl update impl update Connection impl update ResponseRowFormat.java instead of QueryResultRowFormat.java add logic to queryRpc update logic --- .../clirr-ignored-differences.xml | 7 +- .../com/google/cloud/bigquery/BigQuery.java | 28 ++ .../cloud/bigquery/BigQueryDryRunResult.java | 29 ++ .../google/cloud/bigquery/BigQueryImpl.java | 11 + .../cloud/bigquery/BigQueryResultSet.java | 11 +- .../cloud/bigquery/BigQueryResultSetImpl.java | 45 +++ .../com/google/cloud/bigquery/Connection.java | 61 ++++ .../google/cloud/bigquery/ConnectionImpl.java | 208 ++++++++++++ .../cloud/bigquery/ConnectionSettings.java | 316 ++++++++++++++++++ .../cloud/bigquery/QueryConnection.java | 170 ---------- .../ReadClientConnectionConfiguration.java | 20 +- .../cloud/bigquery/ResponseRowFormat.java | 23 ++ 12 files changed, 748 insertions(+), 181 deletions(-) create mode 100644 google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResult.java create mode 100644 google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java create mode 100644 google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java create mode 100644 google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java create mode 100644 google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java delete mode 100644 google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryConnection.java create mode 100644 google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ResponseRowFormat.java diff --git a/google-cloud-bigquery/clirr-ignored-differences.xml b/google-cloud-bigquery/clirr-ignored-differences.xml index 4d66f0620..9505b2699 100644 --- a/google-cloud-bigquery/clirr-ignored-differences.xml +++ b/google-cloud-bigquery/clirr-ignored-differences.xml @@ -5,11 +5,6 @@ 7012 com/google/cloud/bigquery/BigQuery - boolean delete(com.google.cloud.bigquery.JobId) - - - 7012 - com/google/cloud/bigquery/spi/v2/BigQueryRpc - boolean deleteJob(java.lang.String, java.lang.String, java.lang.String) + com.google.cloud.bigquery.Connection createConnection(com.google.cloud.bigquery.ConnectionSettings) \ No newline at end of file diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java index 4e88f000f..1d91cb312 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java @@ -760,6 +760,34 @@ public int hashCode() { */ Job create(JobInfo jobInfo, JobOption... options); + /** + * Creates a new BigQuery query connection. + * + *

Example of creating a query connection. + * + * 

+   * {
+   *   @code
+   *       ConnectionSettings connectionSettings =
+   *         ConnectionSettings.newBuilder()
+   *             .setRequestTimeout(10L)
+   *             .setMaxResults(100L)
+   *             .setUseQueryCache(true)
+   *             .setResponseRowFormat(ResponseRowFormat.GenericRecord)
+   *             .build();
+   *       Connection connection = bigquery.createConnection(connectionSettings);
+   *       try {
+   *           connection.executeSelect("SELECT 1");
+   *       } catch (BigQuerySQLException ex) {
+   *           // handle exception
+   *       }
+   * }
+   *
+ * + * @throws BigQueryException upon failure + */ + Connection createConnection(ConnectionSettings connectionSettings); + /** * Returns the requested dataset or {@code null} if not found. * diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResult.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResult.java new file mode 100644 index 000000000..99f281b16 --- /dev/null +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResult.java @@ -0,0 +1,29 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import com.google.api.services.bigquery.model.QueryParameter; +import java.util.List; + +public interface BigQueryDryRunResult { + + /** Returns the schema of the results. Null if the schema is not supplied. */ + Schema getSchema() throws BigQuerySQLException; + + /** Returns query parameters for standard SQL queries */ + List getQueryParameters() throws BigQuerySQLException; +} diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java index 03635a89e..ceae9feef 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java @@ -345,6 +345,17 @@ public JobId get() { return create(jobInfo, idProvider, options); } + @Override + public Connection createConnection(ConnectionSettings connectionSettings) + throws BigQueryException { + switch (connectionSettings.getResponseRowFormat()) { + default: + // TODO: DETERMINE DEFAULT ROW SERIALIZATION FORMAT + return new ConnectionImpl<>( + connectionSettings, getOptions(), bigQueryRpc, DEFAULT_RETRY_CONFIG); + } + } + @InternalApi("visible for testing") Job create(JobInfo jobInfo, Supplier idProvider, JobOption... options) { boolean idRandom = false; diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java index 0edd0fd25..073d84b56 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java @@ -16,9 +16,9 @@ package com.google.cloud.bigquery; -public interface BigQueryResultSet { +public interface BigQueryResultSet { - /** Returns the schema of the results. Null if the schema is not supplied. */ + /** Returns the schema of the results. */ Schema getSchema(); /** @@ -27,6 +27,9 @@ public interface BigQueryResultSet { */ long getTotalRows(); - /** Returns the next page of result set. Null if there is no more pages left. */ - BigQueryResultSet getNextPage(); + /** + * Returns the next row in the user-specified format. Default format is Avro. Null if there is no + * more rows left. + */ + T getNext(); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java new file mode 100644 index 000000000..bfe93fe84 --- /dev/null +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -0,0 +1,45 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +public class BigQueryResultSetImpl implements BigQueryResultSet { + + private final Schema schema; + private final long totalRows; + private final T nextRow; + + public BigQueryResultSetImpl(Schema schema, long totalRows, T nextRow) { + this.schema = schema; + this.totalRows = totalRows; + this.nextRow = nextRow; + } + + @Override + public Schema getSchema() { + return schema; + } + + @Override + public long getTotalRows() { + return totalRows; + } + + @Override + public T getNext() { + return null; + } +} diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java new file mode 100644 index 000000000..30efe8e96 --- /dev/null +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java @@ -0,0 +1,61 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import com.google.api.services.bigquery.model.QueryParameter; +import java.util.List; +import java.util.Map; + +/** + * A Connection is a session between a Java application and BigQuery. SQL statements are executed + * and results are returned within the context of a connection. + */ +public interface Connection { + + /** Sends a query cancel request. This call will return immediately */ + Boolean cancel() throws BigQuerySQLException; + + /** + * Execute a query dry run that does not return any BigQueryResultSet // TODO: explain more about + * what this method does here + * + * @param sql typically a static SQL SELECT statement + * @exception BigQuerySQLException if a database access error occurs + */ + BigQueryDryRunResult dryRun(String sql) throws BigQuerySQLException; + + /** + * Execute a SQL statement that returns a single BigQueryResultSet + * + * @param sql a static SQL SELECT statement + * @return a BigQueryResultSet that contains the data produced by the query + * @exception BigQuerySQLException if a database access error occurs + */ + BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException; + + /** + * Execute a SQL statement with query parameters that returns a single BigQueryResultSet + * + * @param sql typically a static SQL SELECT statement + * @param parameters named or positional parameters + * @return a BigQueryResultSet that contains the data produced by the query + * @exception BigQuerySQLException if a database access error occurs + */ + BigQueryResultSet executeSelect( + String sql, List parameters, Map labels) + throws BigQuerySQLException; +} diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java new file mode 100644 index 000000000..415fcb2c0 --- /dev/null +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -0,0 +1,208 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import com.google.api.services.bigquery.model.QueryParameter; +import com.google.api.services.bigquery.model.QueryRequest; +import com.google.api.services.bigquery.model.QueryResponse; +import com.google.cloud.bigquery.spi.v2.BigQueryRpc; +import com.google.common.collect.Lists; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.Callable; + +/** Implementation for {@link Connection}, the generic BigQuery connection API (not JDBC). */ +final class ConnectionImpl implements Connection { + + private ConnectionSettings connectionSettings; + private BigQueryOptions options; + private BigQueryRpc bigQueryRpc; + private BigQueryRetryConfig retryConfig; + + ConnectionImpl( + ConnectionSettings connectionSettings, + BigQueryOptions options, + BigQueryRpc bigQueryRpc, + BigQueryRetryConfig retryConfig) { + this.connectionSettings = connectionSettings; + this.options = options; + this.bigQueryRpc = bigQueryRpc; + this.retryConfig = retryConfig; + } + + @Override + public Boolean cancel() throws BigQuerySQLException { + return null; + } + + @Override + public BigQueryDryRunResult dryRun(String sql) throws BigQuerySQLException { + return null; + } + + @Override + public BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException { + // use jobs.query if all properties of the Connection are supported + if (isFastQuerySupported()) { + String projectId = options.getProjectId(); + QueryRequest queryRequest = createQueryRequest(connectionSettings, sql, null, null); + return queryRpc(projectId, queryRequest); + } + // use jobs.insert otherwise + return null; + } + + @Override + public BigQueryResultSet executeSelect( + String sql, List queryParameters, Map labels) + throws BigQuerySQLException { + if (isFastQuerySupported()) { + final String projectId = options.getProjectId(); + final QueryRequest preparedQueryRequest = + createQueryRequest(connectionSettings, sql, queryParameters, labels); + // return queryRpc(projectId, preparedQueryRequest); + } + return null; + } + + private BigQueryResultSet queryRpc(final String projectId, final QueryRequest queryRequest) { + com.google.api.services.bigquery.model.QueryResponse results; + try { + results = + BigQueryRetryHelper.runWithRetries( + new Callable() { + @Override + public com.google.api.services.bigquery.model.QueryResponse call() { + return bigQueryRpc.queryRpc(projectId, queryRequest); + } + }, + options.getRetrySettings(), + BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, + options.getClock(), + retryConfig); + } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) { + throw BigQueryException.translateAndThrow(e); + } + if (results.getErrors() != null) { + List bigQueryErrors = + Lists.transform(results.getErrors(), BigQueryError.FROM_PB_FUNCTION); + // Throwing BigQueryException since there may be no JobId and we want to stay consistent + // with the case where there there is a HTTP error + throw new BigQueryException(bigQueryErrors); + } + + long numRows; + Schema schema; + if (results.getJobComplete() && results.getSchema() != null) { + schema = Schema.fromPb(results.getSchema()); + numRows = results.getTotalRows().longValue(); + } else { + // Query is long running (> 10s) and hasn't completed yet, or query completed but didn't + // return the schema, fallback. Some operations don't return the schema and can be optimized + // here, but this is left as future work. + JobId jobId = JobId.fromPb(results.getJobReference()); + return null; + } + + // TODO(prasmish): WRITE A PRIVATE METHOD TO CACHE FIRST PAGE + // cacheFirstPage(...) + + if (results.getPageToken() != null) { + JobId jobId = JobId.fromPb(results.getJobReference()); + return new BigQueryResultSetImpl<>( + schema, numRows, null + /** iterate(cachedFirstPage)* */ + ); + } + + // only 1 page of result + // TODO(prasmish): GET NEXT ROW AND PASS INTO BigQueryResultSetImpl + + // Query is long running (> 10s) and hasn't completed yet or has more than 1 pages of results + // Use tabledata.list or high through-put Read API to fetch the rest of the pages + JobId jobId = JobId.fromPb(results.getJobReference()); + if (useReadAPI(results)) { + // Discard the first page and use BigQuery Storage Read API to fetch the entire result set + + } else { + // Use tabledata.list to fetch the rest of the result set + + } + return null; + } + + private boolean isFastQuerySupported() { + return connectionSettings.getClustering() == null + && connectionSettings.getCreateDisposition() == null + && connectionSettings.getDestinationEncryptionConfiguration() == null + && connectionSettings.getDestinationTable() == null + && connectionSettings.getJobTimeoutMs() == null + && connectionSettings.getMaximumBillingTier() == null + && connectionSettings.getPriority() == null + && connectionSettings.getRangePartitioning() == null + && connectionSettings.getSchemaUpdateOptions() == null + && connectionSettings.getTableDefinitions() == null + && connectionSettings.getTimePartitioning() == null + && connectionSettings.getUserDefinedFunctions() == null + && connectionSettings.getWriteDisposition() == null; + } + + private boolean useReadAPI(QueryResponse response) { + long totalRows = response.getTotalRows().longValue(); + long pageRows = response.getRows().size(); + long resultRatio = totalRows / pageRows; + return resultRatio + > connectionSettings + .getReadClientConnectionConfiguration() + .getTotalToPageRowCountRatio() + && totalRows > connectionSettings.getReadClientConnectionConfiguration().getMinResultSize(); + } + + private QueryRequest createQueryRequest( + ConnectionSettings connectionSettings, + String sql, + List queryParameters, + Map labels) { + QueryRequest content = new QueryRequest(); + String requestId = UUID.randomUUID().toString(); + + if (connectionSettings.getConnectionProperties() != null) { + content.setConnectionProperties( + Lists.transform( + connectionSettings.getConnectionProperties(), ConnectionProperty.TO_PB_FUNCTION)); + } + if (connectionSettings.getDefaultDataset() != null) { + content.setDefaultDataset(connectionSettings.getDefaultDataset().toPb()); + } + if (connectionSettings.getMaximumBytesBilled() != null) { + content.setMaximumBytesBilled(connectionSettings.getMaximumBytesBilled()); + } + if (connectionSettings.getMaxResults() != null) { + content.setMaxResults(connectionSettings.getMaxResults()); + } + if (queryParameters != null) { + content.setQueryParameters(queryParameters); + } + if (labels != null) { + content.setLabels(labels); + } + content.setQuery(sql); + content.setRequestId(requestId); + return content; + } +} diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java new file mode 100644 index 000000000..6fe0cbce9 --- /dev/null +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java @@ -0,0 +1,316 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import com.google.auto.value.AutoValue; +import com.google.cloud.bigquery.JobInfo.CreateDisposition; +import com.google.cloud.bigquery.JobInfo.SchemaUpdateOption; +import com.google.cloud.bigquery.JobInfo.WriteDisposition; +import com.google.cloud.bigquery.QueryJobConfiguration.Priority; +import java.util.List; +import java.util.Map; + +/** ConnectionSettings for setting up a BigQuery query connection. */ +@AutoValue +public abstract class ConnectionSettings { + ConnectionSettings() { + // Package private so users can't subclass it but AutoValue can. + } + + /** Returns the synchronous response timeoutMs associated with this query */ + public abstract Long getRequestTimeout(); + + /** Returns the connection properties for connection string with this query */ + public abstract List getConnectionProperties(); + + /** Returns the default dataset */ + public abstract DatasetId getDefaultDataset(); + + /** Returns the limits the bytes billed for this job */ + public abstract Long getMaximumBytesBilled(); + + /** Returns the maximum number of rows of data */ + public abstract Long getMaxResults(); + + /** Returns whether to look for the result in the query cache */ + public abstract Boolean getUseQueryCache(); + + /** Returns the BigQuery Storage read API configuration */ + public abstract ReadClientConnectionConfiguration getReadClientConnectionConfiguration(); + + /** Returns the return row serialization format */ + public abstract ResponseRowFormat getResponseRowFormat(); + + /** + * Below properties are only supported by jobs.insert API and not yet supported by jobs.query API + * * + */ + + /** Returns the clustering specification for the destination table. */ + public abstract Clustering getClustering(); + + /** + * Returns whether the job is allowed to create new tables. + * + * @see + * Create Disposition + */ + public abstract CreateDisposition getCreateDisposition(); + + /** Returns the custom encryption configuration (e.g., Cloud KMS keys) */ + public abstract EncryptionConfiguration getDestinationEncryptionConfiguration(); + + /** + * Returns the table where to put query results. If not provided a new table is created. This + * value is required if {@link #allowLargeResults()} is {@code true}. + */ + public abstract TableId getDestinationTable(); + + /** Returns the timeout associated with this job */ + public abstract Long getJobTimeoutMs(); + + /** Returns the optional billing tier limit for this job. */ + public abstract Integer getMaximumBillingTier(); + + /** Returns the query priority. */ + public abstract Priority getPriority(); + + /** Returns the range partitioning specification for the table */ + public abstract RangePartitioning getRangePartitioning(); + + /** + * [Experimental] Returns options allowing the schema of the destination table to be updated as a + * side effect of the query job. Schema update options are supported in two cases: when + * writeDisposition is WRITE_APPEND; when writeDisposition is WRITE_TRUNCATE and the destination + * table is a partition of a table, specified by partition decorators. For normal tables, + * WRITE_TRUNCATE will always overwrite the schema. + */ + public abstract List getSchemaUpdateOptions(); + + /** + * Returns the external tables definitions. If querying external data sources outside of BigQuery, + * this value describes the data format, location and other properties of the data sources. By + * defining these properties, the data sources can be queried as if they were standard BigQuery + * tables. + */ + public abstract Map getTableDefinitions(); + + /** Returns the time partitioning specification for the destination table. */ + public abstract TimePartitioning getTimePartitioning(); + + /** + * Returns user defined function resources that can be used by this query. Function resources can + * either be defined inline ({@link UserDefinedFunction.Type#INLINE}) or loaded from a Google + * Cloud Storage URI ({@link UserDefinedFunction.Type#FROM_URI}. + */ + public abstract List getUserDefinedFunctions(); + + /** + * Returns the action that should occur if the destination table already exists. + * + * @see + * Write Disposition + */ + public abstract WriteDisposition getWriteDisposition(); + + /** Returns a builder pre-populated using the current values of this field. */ + public abstract Builder toBuilder(); + + /** Returns a builder for a {@code ConnectionSettings} object. */ + public static Builder newBuilder() { + return new AutoValue_ConnectionSettings.Builder(); + } + + @AutoValue.Builder + public abstract static class Builder { + + /** + * Sets how long to wait for the query to complete, in milliseconds, before the request times + * out and returns. Note that this is only a timeout for the request, not the query. If the + * query takes longer to run than the timeout value, the call returns without any results and + * with the 'jobComplete' flag set to false. You can call GetQueryResults() to wait for the + * query to complete and read the results. The default value is 10000 milliseconds (10 seconds). + * + * @param timeoutMs or {@code null} for none + */ + public abstract Builder setRequestTimeout(Long timeoutMs); + + /** + * Sets a connection-level property to customize query behavior. + * + * @param connectionProperties connectionProperties or {@code null} for none + */ + public abstract Builder setConnectionProperties(List connectionProperties); + + /** + * Sets the default dataset. This dataset is used for all unqualified table names used in the + * query. + */ + public abstract Builder setDefaultDataset(DatasetId datasetId); + + /** + * Limits the bytes billed for this job. Queries that will have bytes billed beyond this limit + * will fail (without incurring a charge). If unspecified, this will be set to your project + * default. + * + * @param maximumBytesBilled maximum bytes billed for this job + */ + public abstract Builder setMaximumBytesBilled(Long maximumBytesBilled); + + /** + * Sets the maximum number of rows of data to return per page of results. Setting this flag to a + * small value such as 1000 and then paging through results might improve reliability when the + * query result set is large. In addition to this limit, responses are also limited to 10 MB. By + * default, there is no maximum row count, and only the byte limit applies. + * + * @param maxResults maxResults or {@code null} for none + */ + public abstract Builder setMaxResults(Long maxResults); + + /** + * Sets whether to look for the result in the query cache. The query cache is a best-effort + * cache that will be flushed whenever tables in the query are modified. Moreover, the query + * cache is only available when {@link + * QueryJobConfiguration.Builder#setDestinationTable(TableId)} is not set. + * + * @see Query Caching + */ + public abstract Builder setUseQueryCache(Boolean useQueryCache); + + /** + * Sets the values necessary to determine whether table result will be read using the BigQuery + * Storage client Read API. The BigQuery Storage client Read API will be used to read the query + * result when the totalToFirstPageSizeRatio (default 3) and minimumTableSize (default 100MB) + * conditions set are met. A ReadSession will be created using Apache Avro data format for + * serialization. + * + *

It also sets the maximum number of table rows allowed in buffer before streaming them to + * the BigQueryResultSet. + * + * @param readClientConnectionConfiguration or {@code null} for none + */ + public abstract Builder setReadClientConnectionConfiguration( + ReadClientConnectionConfiguration readClientConnectionConfiguration); + + /** + * Sets the row format received from the BigQuery API. + * + * @param responseRowFormat or {@code null} for none + */ + public abstract Builder setResponseRowFormat(ResponseRowFormat responseRowFormat); + + /** Sets the clustering specification for the destination table. */ + public abstract Builder setClustering(Clustering clustering); + + /** + * Sets whether the job is allowed to create tables. + * + * @see + * Create Disposition + */ + public abstract Builder setCreateDisposition(CreateDisposition createDisposition); + + /** + * Sets the custom encryption configuration (e.g., Cloud KMS keys). + * + * @param destinationEncryptionConfiguration destinationEncryptionConfiguration or {@code null} + * for none + */ + public abstract Builder setDestinationEncryptionConfiguration( + EncryptionConfiguration destinationEncryptionConfiguration); + + /** + * Sets the table where to put query results. If not provided a new table is created. This value + * is required if {@link QueryJobConfiguration.Builder#setAllowLargeResults(Boolean)} is set to + * {@code true}. + */ + public abstract Builder setDestinationTable(TableId destinationTable); + + /** + * [Optional] Job timeout in milliseconds. If this time limit is exceeded, BigQuery may attempt + * to terminate the job. + * + * @param jobTimeoutMs jobTimeoutMs or {@code null} for none + */ + public abstract Builder setJobTimeoutMs(Long jobTimeoutMs); + + /** + * Limits the billing tier for this job. Queries that have resource usage beyond this tier will + * fail (without incurring a charge). If unspecified, this will be set to your project default. + * + * @param maximumBillingTier maximum billing tier for this job + */ + public abstract Builder setMaximumBillingTier(Integer maximumBillingTier); + + /** + * Sets a priority for the query. If not specified the priority is assumed to be {@link + * Priority#INTERACTIVE}. + */ + public abstract Builder setPriority(Priority priority); + + /** + * Range partitioning specification for this table. Only one of timePartitioning and + * rangePartitioning should be specified. + * + * @param rangePartitioning rangePartitioning or {@code null} for none + */ + public abstract Builder setRangePartitioning(RangePartitioning rangePartitioning); + + /** + * [Experimental] Sets options allowing the schema of the destination table to be updated as a + * side effect of the query job. Schema update options are supported in two cases: when + * writeDisposition is WRITE_APPEND; when writeDisposition is WRITE_TRUNCATE and the destination + * table is a partition of a table, specified by partition decorators. For normal tables, + * WRITE_TRUNCATE will always overwrite the schema. + */ + public abstract Builder setSchemaUpdateOptions(List schemaUpdateOptions); + + /** + * Sets the external tables definitions. If querying external data sources outside of BigQuery, + * this value describes the data format, location and other properties of the data sources. By + * defining these properties, the data sources can be queried as if they were standard BigQuery + * tables. + */ + public abstract Builder setTableDefinitions( + Map tableDefinitions); + + /** Sets the time partitioning specification for the destination table. */ + public abstract Builder setTimePartitioning(TimePartitioning timePartitioning); + + /** + * Sets user defined function resources that can be used by this query. Function resources can + * either be defined inline ({@link UserDefinedFunction#inline(String)}) or loaded from a Google + * Cloud Storage URI ({@link UserDefinedFunction#fromUri(String)}. + */ + public abstract Builder setUserDefinedFunctions(List userDefinedFunctions); + + /** + * Sets the action that should occur if the destination table already exists. + * + * @see + * Write Disposition + */ + public abstract Builder setWriteDisposition(WriteDisposition writeDisposition); + + /** Creates a {@code ConnectionSettings} object. */ + public abstract ConnectionSettings build(); + } +} diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryConnection.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryConnection.java deleted file mode 100644 index d3cb6f81b..000000000 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryConnection.java +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.bigquery; - -import com.google.api.services.bigquery.model.QueryParameter; -import java.util.List; -import java.util.Map; - -/** - * A Connection is a session between a Java application and BigQuery. SQL statements are executed - * and results are returned within the context of a connection. - */ -public interface QueryConnection { - - /** - * Sets how long to wait for the query to complete, in milliseconds, before the request times out - * and returns. Note that this is only a timeout for the request, not the query. If the query - * takes longer to run than the timeout value, the call returns without any results and with the - * 'jobComplete' flag set to false. You can call GetQueryResults() to wait for the query to - * complete and read the results. The default value is 10000 milliseconds (10 seconds). - * - * @param timeoutMs or {@code null} for none - */ - void setSynchronousResponseTimeoutMs(Long timeoutMs) throws BigQuerySQLException; - - /** Returns the synchronous response timeoutMs associated with this query */ - Long getSynchronousResponseTimeoutMs() throws BigQuerySQLException; - - /** - * Sets a connection-level property to customize query behavior. Under JDBC, these correspond - * directly to connection properties passed to the DriverManager. - * - * @param connectionProperties connectionProperties or {@code null} for none - */ - void setConnectionProperties(List connectionProperties) - throws BigQuerySQLException; - - /** Returns the connection properties for connection string with this query */ - List getConnectionProperties() throws BigQuerySQLException; - - /** - * Sets the default dataset. This dataset is used for all unqualified table names used in the - * query. - */ - void setDefaultDataset() throws BigQuerySQLException; - - /** Returns the default dataset */ - DatasetId getDefaultDataset() throws BigQuerySQLException; - - /** - * Sets the labels associated with this query. You can use these to organize and group your - * queries. Label keys and values can be no longer than 63 characters, can only contain lowercase - * letters, numeric characters, underscores and dashes. International characters are allowed. - * Label values are optional. Label keys must start with a letter and each label in the list must - * have a different key. - * - * @param labels labels or {@code null} for none - */ - void setLabels(Map labels) throws BigQuerySQLException; - - /** Returns the labels associated with this query */ - Map getLabels() throws BigQuerySQLException; - - /** Clear the labels associated with this query */ - void cleaLabels() throws BigQuerySQLException; - - /** - * Limits the bytes billed for this job. Queries that will have bytes billed beyond this limit - * will fail (without incurring a charge). If unspecified, this will be set to your project - * default. - * - * @param maximumBytesBilled maximum bytes billed for this job - */ - void setMaximumBytesBilled(Long maximumBytesBilled) throws BigQuerySQLException; - - /** Returns the limits the bytes billed for this job */ - Long getMaximumBytesBilled() throws BigQuerySQLException; - - /** - * Sets the maximum number of rows of data to return per page of results. Setting this flag to a - * small value such as 1000 and then paging through results might improve reliability when the - * query result set is large. In addition to this limit, responses are also limited to 10 MB. By - * default, there is no maximum row count, and only the byte limit applies. - * - * @param maxResults maxResults or {@code null} for none - */ - void setMaxResults(Long maxResults) throws BigQuerySQLException; - - /** Returns the maximum number of rows of data */ - Long getMaxResults() throws BigQuerySQLException; - - /** Returns query parameters for standard SQL queries */ - List getQueryParameters() throws BigQuerySQLException; - - /** - * Sets whether to look for the result in the query cache. The query cache is a best-effort cache - * that will be flushed whenever tables in the query are modified. Moreover, the query cache is - * only available when {@link QueryJobConfiguration.Builder#setDestinationTable(TableId)} is not - * set. - * - * @see Query Caching - */ - void setUseQueryCache(Boolean useQueryCache) throws BigQuerySQLException; - - /** Returns whether to look for the result in the query cache */ - Boolean getUseQueryCache() throws BigQuerySQLException; - - /** - * Execute a query dry run that does not return any BigQueryResultSet - * - * @param sql typically a static SQL SELECT statement - * @exception BigQuerySQLException if a database access error occurs - */ - void dryRun(String sql) throws BigQuerySQLException; - - /** - * Execute a SQL statement that returns a single BigQueryResultSet - * - * @param sql typically a static SQL SELECT statement - * @return a BigQueryResultSet that contains the data produced by the query - * @exception BigQuerySQLException if a database access error occurs - */ - BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException; - - /** - * Execute a SQL statement with query parameters that returns a single BigQueryResultSet - * - * @param sql typically a static SQL SELECT statement - * @param parameters named or positional parameters - * @return a BigQueryResultSet that contains the data produced by the query - * @exception BigQuerySQLException if a database access error occurs - */ - BigQueryResultSet executeSelect(String sql, List parameters) - throws BigQuerySQLException; - - /** - * Sets the values necessary to determine whether table result will be read using the BigQuery - * Storage client Read API. The BigQuery Storage client Read API will be used to read the query - * result when the totalToFirstPageSizeRatio (default 3) and minimumTableSize (default 100MB) - * conditions set are met. A ReadSession will be created using Apache Avro data format for - * serialization. - * - *

It also sets the maximum number of table rows allowed in buffer before streaming them to the - * BigQueryResultSet. - * - * @param readClientConnectionConfiguration or {@code null} for none - * @exception BigQueryException if an error occurs in setting these values - */ - void setReadClientConnectionConfiguration( - ReadClientConnectionConfiguration readClientConnectionConfiguration) throws BigQueryException; -} - -class Parameter { - String name; - QueryParameterValue value; -} diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ReadClientConnectionConfiguration.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ReadClientConnectionConfiguration.java index 8503e642a..7b683b565 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ReadClientConnectionConfiguration.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ReadClientConnectionConfiguration.java @@ -26,13 +26,31 @@ public abstract class ReadClientConnectionConfiguration implements Serializable @AutoValue.Builder public abstract static class Builder { - /* Sets the buffer size during streaming from the BigQueryStorage Read client */ + /** + * Sets the total row count to page row count ratio used to determine whether to us the + * BigQueryStorage Read client to fetch result sets after the first page. + */ + public abstract Builder setTotalToPageRowCountRatio(Long ratio); + + /** + * Sets the minimum number of table rows in the query results used to determine whether to us + * the BigQueryStorage Read client to fetch result sets after the first page. + */ + public abstract Builder setMinResultSize(Long numRows); + + /** Sets the buffer size during streaming from the BigQueryStorage Read client. */ public abstract Builder setBufferSize(Long bufferSize); /** Creates a {@code ReadClientConnectionConfiguration} object. */ public abstract ReadClientConnectionConfiguration build(); } + /** Returns the totalToPageRowCountRatio in this configuration. */ + public abstract Long getTotalToPageRowCountRatio(); + + /** Returns the minResultSize in this configuration. */ + public abstract Long getMinResultSize(); + /** Returns the bufferSize in this configuration. */ public abstract Long getBufferSize(); diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ResponseRowFormat.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ResponseRowFormat.java new file mode 100644 index 000000000..3e03b63e3 --- /dev/null +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ResponseRowFormat.java @@ -0,0 +1,23 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +/** Supported row serialization formats * */ +public enum ResponseRowFormat { + GenericRecord, + JSON +} From 359579990b33cc7c30a977e117001350c63f2dd6 Mon Sep 17 00:00:00 2001 From: stephwang Date: Tue, 17 Aug 2021 14:59:57 -0400 Subject: [PATCH 003/277] update ConnectionImpl add create job and getQueryResults logic update query logic and add endpoint to fetch destinationTable from jobId refactor BigQueryResultSet to return next row as ResultSet refactor code to remove JobOptions Added logic for processing query response result Added shared buffer for the Consumer linting linting linting added next & getString added next & getString added getString method Implemented getString Implemented BlockingQueue for the buffer minor refactor AbstractJdbcResultSet Implementation extending AbstractJdbcResultSet Modified the Connection Interface to return java.sql.ResultSet for executeSelect Modifie the Impl methods in ConnectionImpl to return java.sql.ResultSet, commenting the queryJob logic for the time being as it currently doesn't return the java.sql.ResultSet Moved unimplemented methods to AbstractJdbcResultSet Implemented getters for various datatypes update javadoc --- .../clirr-ignored-differences.xml | 20 + .../cloud/bigquery/AbstractJdbcResultSet.java | 989 ++++++++++++++++++ .../com/google/cloud/bigquery/BigQuery.java | 6 - .../google/cloud/bigquery/BigQueryImpl.java | 7 +- .../cloud/bigquery/BigQueryResultSet.java | 15 +- .../cloud/bigquery/BigQueryResultSetImpl.java | 139 ++- .../com/google/cloud/bigquery/Connection.java | 46 +- .../google/cloud/bigquery/ConnectionImpl.java | 392 +++++-- .../cloud/bigquery/ConnectionSettings.java | 57 +- .../cloud/bigquery/ResponseRowFormat.java | 23 - .../cloud/bigquery/spi/v2/BigQueryRpc.java | 31 + .../bigquery/spi/v2/HttpBigQueryRpc.java | 62 ++ 12 files changed, 1662 insertions(+), 125 deletions(-) create mode 100644 google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/AbstractJdbcResultSet.java delete mode 100644 google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ResponseRowFormat.java diff --git a/google-cloud-bigquery/clirr-ignored-differences.xml b/google-cloud-bigquery/clirr-ignored-differences.xml index 9505b2699..832712adb 100644 --- a/google-cloud-bigquery/clirr-ignored-differences.xml +++ b/google-cloud-bigquery/clirr-ignored-differences.xml @@ -7,4 +7,24 @@ com/google/cloud/bigquery/BigQuery com.google.cloud.bigquery.Connection createConnection(com.google.cloud.bigquery.ConnectionSettings) + + 7012 + com/google/cloud/bigquery/spi/v2/BigQueryRpc + com.google.api.services.bigquery.model.Job createJobForQuery(com.google.api.services.bigquery.model.Job) + + + 7012 + com/google/cloud/bigquery/spi/v2/BigQueryRpc + com.google.api.services.bigquery.model.Job getQueryJob(java.lang.String, java.lang.String, java.lang.String) + + + 7012 + com/google/cloud/bigquery/spi/v2/BigQueryRpc + com.google.api.services.bigquery.model.GetQueryResultsResponse getQueryResultsWithRowLimit(java.lang.String, java.lang.String, java.lang.String, java.lang.Long) + + + 7012 + com/google/cloud/bigquery/spi/v2/BigQueryRpc + com.google.api.services.bigquery.model.TableDataList listTableDataWithRowLimit(java.lang.String, java.lang.String, java.lang.String, java.lang.Long) + \ No newline at end of file diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/AbstractJdbcResultSet.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/AbstractJdbcResultSet.java new file mode 100644 index 000000000..4456317f2 --- /dev/null +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/AbstractJdbcResultSet.java @@ -0,0 +1,989 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import java.io.InputStream; +import java.io.Reader; +import java.math.BigDecimal; +import java.net.URL; +import java.sql.*; +import java.util.Calendar; +import java.util.Map; + +abstract class AbstractJdbcResultSet implements ResultSet { + + @Override + public BigDecimal getBigDecimal(String columnLabel, int scale) throws SQLException { + // TODO: Implement the logic + throw new RuntimeException("Not implemented"); + } + + @Override + public void close() throws SQLException { + // TODO: Implement the logic + throw new RuntimeException("Not implemented"); + } + + @Override + public boolean wasNull() throws SQLException { + // TODO: Implement the logic + throw new RuntimeException("Not implemented"); + } + + @Override + public String getString(int columnIndex) throws SQLException { + // TODO: Implement the logic + throw new RuntimeException("Not implemented"); + } + + @Override + public boolean getBoolean(int columnIndex) throws SQLException { + // TODO: Implement the logic + throw new RuntimeException("Not implemented"); + } + + @Override + public byte getByte(int columnIndex) throws SQLException { + // TODO: Implement the logic + throw new RuntimeException("Not implemented"); + } + + @Override + public short getShort(int columnIndex) throws SQLException { + // TODO: Implement the logic + throw new RuntimeException("Not implemented"); + } + + @Override + public int getInt(int columnIndex) throws SQLException { + // TODO: Implement the logic + throw new RuntimeException("Not implemented"); + } + + @Override + public long getLong(int columnIndex) throws SQLException { + // TODO: Implement the logic + throw new RuntimeException("Not implemented"); + } + + @Override + public float getFloat(int columnIndex) throws SQLException { + // TODO: Implement the logic + throw new RuntimeException("Not implemented"); + } + + @Override + public double getDouble(int columnIndex) throws SQLException { + // TODO: Implement the logic + throw new RuntimeException("Not implemented"); + } + + @Override + public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException { + // TODO: Implement the logic + throw new RuntimeException("Not implemented"); + } + + @Override + public byte[] getBytes(int columnIndex) throws SQLException { + // TODO: Implement the logic + throw new RuntimeException("Not implemented"); + } + + @Override + public Date getDate(int columnIndex) throws SQLException { + // TODO: Implement the logic + throw new RuntimeException("Not implemented"); + } + + @Override + public Time getTime(int columnIndex) throws SQLException { + // TODO: Implement the logic + throw new RuntimeException("Not implemented"); + } + + @Override + public Timestamp getTimestamp(int columnIndex) throws SQLException { + // TODO: Implement the logic + throw new RuntimeException("Not implemented"); + } + + @Override + public InputStream getAsciiStream(int columnIndex) throws SQLException { + // TODO: Implement the logic + throw new RuntimeException("Not implemented"); + } + + @Override + public InputStream getUnicodeStream(int columnIndex) throws SQLException { + // TODO: Implement the logic + throw new RuntimeException("Not implemented"); + } + + @Override + public InputStream getBinaryStream(int columnIndex) throws SQLException { + // TODO: Implement the logic + throw new RuntimeException("Not implemented"); + } + + @Override + public int findColumn(String columnLabel) throws SQLException { + // TODO: Implement the logic + throw new RuntimeException("Not implemented"); + } + + @Override + public Reader getCharacterStream(int columnIndex) throws SQLException { + // TODO: Implement the logic + throw new RuntimeException("Not implemented"); + } + + @Override + public Reader getCharacterStream(String columnLabel) throws SQLException { + // TODO: Implement the logic + throw new RuntimeException("Not implemented"); + } + + @Override + public BigDecimal getBigDecimal(int columnIndex) throws SQLException { + // TODO: Implement the logic + throw new RuntimeException("Not implemented"); + } + + @Override + public SQLWarning getWarnings() throws SQLException { + return null; + } + + @Override + public void clearWarnings() throws SQLException {} + + @Override + public String getCursorName() throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public boolean isLast() throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void beforeFirst() throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void afterLast() throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public boolean first() throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public boolean last() throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public boolean absolute(int row) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public boolean relative(int rows) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public boolean previous() throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void setFetchDirection(int direction) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public int getFetchDirection() throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void setFetchSize(int rows) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public int getFetchSize() throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public int getType() throws SQLException { + return TYPE_FORWARD_ONLY; + } + + @Override + public int getConcurrency() throws SQLException { + return CONCUR_READ_ONLY; + } + + @Override + public boolean rowUpdated() throws SQLException { + return false; + } + + @Override + public boolean rowInserted() throws SQLException { + return false; + } + + @Override + public boolean rowDeleted() throws SQLException { + return false; + } + + @Override + public void updateNull(int columnIndex) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateBoolean(int columnIndex, boolean x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateByte(int columnIndex, byte x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateShort(int columnIndex, short x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateInt(int columnIndex, int x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateLong(int columnIndex, long x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateFloat(int columnIndex, float x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateDouble(int columnIndex, double x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateBigDecimal(int columnIndex, BigDecimal x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateString(int columnIndex, String x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateBytes(int columnIndex, byte[] x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateDate(int columnIndex, Date x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateTime(int columnIndex, Time x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateTimestamp(int columnIndex, Timestamp x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateAsciiStream(int columnIndex, InputStream x, int length) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateBinaryStream(int columnIndex, InputStream x, int length) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateCharacterStream(int columnIndex, Reader x, int length) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateObject(int columnIndex, Object x, int scaleOrLength) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateObject(int columnIndex, Object x, SQLType type, int scaleOrLength) + throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateObject(int columnIndex, Object x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateObject(int columnIndex, Object x, SQLType type) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateNull(String columnLabel) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateBoolean(String columnLabel, boolean x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateByte(String columnLabel, byte x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateShort(String columnLabel, short x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateInt(String columnLabel, int x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateLong(String columnLabel, long x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateFloat(String columnLabel, float x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateDouble(String columnLabel, double x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateBigDecimal(String columnLabel, BigDecimal x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateString(String columnLabel, String x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateBytes(String columnLabel, byte[] x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateDate(String columnLabel, Date x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateTime(String columnLabel, Time x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateTimestamp(String columnLabel, Timestamp x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateAsciiStream(String columnLabel, InputStream x, int length) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateBinaryStream(String columnLabel, InputStream x, int length) + throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateCharacterStream(String columnLabel, Reader reader, int length) + throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateObject(String columnLabel, Object x, int scaleOrLength) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateObject(String columnLabel, Object x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateObject(String columnLabel, Object x, SQLType type) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateObject(String columnLabel, Object x, SQLType type, int scaleOrLength) + throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void insertRow() throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateRow() throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void deleteRow() throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void refreshRow() throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void cancelRowUpdates() throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void moveToInsertRow() throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void moveToCurrentRow() throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public Ref getRef(int columnIndex) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public Ref getRef(String columnLabel) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateRef(int columnIndex, Ref x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateRef(String columnLabel, Ref x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateBlob(int columnIndex, Blob x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateBlob(String columnLabel, Blob x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateClob(int columnIndex, Clob x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateClob(String columnLabel, Clob x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateArray(int columnIndex, Array x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateArray(String columnLabel, Array x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public RowId getRowId(int columnIndex) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public RowId getRowId(String columnLabel) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateRowId(int columnIndex, RowId x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateRowId(String columnLabel, RowId x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateNString(int columnIndex, String nString) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateNString(String columnLabel, String nString) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateNClob(int columnIndex, NClob nClob) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateNClob(String columnLabel, NClob nClob) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public SQLXML getSQLXML(int columnIndex) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public SQLXML getSQLXML(String columnLabel) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateSQLXML(int columnIndex, SQLXML xmlObject) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateSQLXML(String columnLabel, SQLXML xmlObject) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateNCharacterStream(int columnIndex, Reader x, long length) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateNCharacterStream(String columnLabel, Reader reader, long length) + throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateAsciiStream(int columnIndex, InputStream x, long length) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateBinaryStream(int columnIndex, InputStream x, long length) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateCharacterStream(int columnIndex, Reader x, long length) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateAsciiStream(String columnLabel, InputStream x, long length) + throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateBinaryStream(String columnLabel, InputStream x, long length) + throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateCharacterStream(String columnLabel, Reader reader, long length) + throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateBlob(int columnIndex, InputStream inputStream, long length) + throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateBlob(String columnLabel, InputStream inputStream, long length) + throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateClob(int columnIndex, Reader reader, long length) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateClob(String columnLabel, Reader reader, long length) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateNClob(int columnIndex, Reader reader, long length) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateNClob(String columnLabel, Reader reader, long length) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateNCharacterStream(int columnIndex, Reader x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateNCharacterStream(String columnLabel, Reader reader) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateAsciiStream(int columnIndex, InputStream x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateBinaryStream(int columnIndex, InputStream x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateCharacterStream(int columnIndex, Reader x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateAsciiStream(String columnLabel, InputStream x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateBinaryStream(String columnLabel, InputStream x) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateCharacterStream(String columnLabel, Reader reader) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateBlob(int columnIndex, InputStream inputStream) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateBlob(String columnLabel, InputStream inputStream) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateClob(int columnIndex, Reader reader) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateClob(String columnLabel, Reader reader) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateNClob(int columnIndex, Reader reader) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public void updateNClob(String columnLabel, Reader reader) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public boolean isBeforeFirst() throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public boolean isAfterLast() throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public boolean isFirst() throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public int getRow() throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public Statement getStatement() throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public Object getObject(int columnIndex, Map> map) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public Blob getBlob(int columnIndex) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public Clob getClob(int columnIndex) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public Array getArray(int columnIndex) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public float getFloat(String columnLabel) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public Object getObject(String columnLabel, Map> map) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public Blob getBlob(String columnLabel) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public Clob getClob(String columnLabel) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public Array getArray(String columnLabel) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public Date getDate(int columnIndex, Calendar cal) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public Date getDate(String columnLabel, Calendar cal) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public Time getTime(int columnIndex, Calendar cal) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public Time getTime(String columnLabel, Calendar cal) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public Timestamp getTimestamp(int columnIndex, Calendar cal) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public Timestamp getTimestamp(String columnLabel, Calendar cal) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public URL getURL(int columnIndex) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public URL getURL(String columnLabel) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public int getHoldability() throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public boolean isClosed() throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public NClob getNClob(int columnIndex) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public NClob getNClob(String columnLabel) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public String getNString(int columnIndex) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public String getNString(String columnLabel) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public Reader getNCharacterStream(int columnIndex) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public Reader getNCharacterStream(String columnLabel) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public T getObject(int columnIndex, Class type) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public T getObject(String columnLabel, Class type) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public short getShort(String columnLabel) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public int getInt(String columnLabel) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public byte[] getBytes(String columnLabel) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public Date getDate(String columnLabel) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public Time getTime(String columnLabel) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public InputStream getAsciiStream(String columnLabel) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public InputStream getUnicodeStream(String columnLabel) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public InputStream getBinaryStream(String columnLabel) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public ResultSetMetaData getMetaData() throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public Object getObject(int columnIndex) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public T unwrap(Class iface) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } + + @Override + public boolean isWrapperFor(Class iface) throws SQLException { + throw new SQLFeatureNotSupportedException(); + } +} diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java index 1d91cb312..e83cb6f18 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java @@ -773,14 +773,8 @@ public int hashCode() { * .setRequestTimeout(10L) * .setMaxResults(100L) * .setUseQueryCache(true) - * .setResponseRowFormat(ResponseRowFormat.GenericRecord) * .build(); * Connection connection = bigquery.createConnection(connectionSettings); - * try { - * connection.executeSelect("SELECT 1"); - * } catch (BigQuerySQLException ex) { - * // handle exception - * } * } * * diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java index ceae9feef..248e1deda 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java @@ -348,12 +348,7 @@ public JobId get() { @Override public Connection createConnection(ConnectionSettings connectionSettings) throws BigQueryException { - switch (connectionSettings.getResponseRowFormat()) { - default: - // TODO: DETERMINE DEFAULT ROW SERIALIZATION FORMAT - return new ConnectionImpl<>( - connectionSettings, getOptions(), bigQueryRpc, DEFAULT_RETRY_CONFIG); - } + return new ConnectionImpl(connectionSettings, getOptions(), bigQueryRpc, DEFAULT_RETRY_CONFIG); } @InternalApi("visible for testing") diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java index 073d84b56..7a3277c3a 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java @@ -16,6 +16,8 @@ package com.google.cloud.bigquery; +import java.sql.SQLException; + public interface BigQueryResultSet { /** Returns the schema of the results. */ @@ -27,9 +29,12 @@ public interface BigQueryResultSet { */ long getTotalRows(); - /** - * Returns the next row in the user-specified format. Default format is Avro. Null if there is no - * more rows left. - */ - T getNext(); + /** Returns the next row. Null if there is no more rows left. */ + // ResultSet getNext(); + + /*Advances the result set to the next row, returning false if no such row exists. Potentially blocking operation*/ + boolean next() throws SQLException; + + /*Returns the value of a String field if the field exists, otherwise returns null*/ + String getString(String fieldName) throws SQLException; } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index bfe93fe84..faaacd835 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -16,16 +16,30 @@ package com.google.cloud.bigquery; -public class BigQueryResultSetImpl implements BigQueryResultSet { +import com.google.api.services.bigquery.model.TableRow; +import java.math.BigDecimal; +import java.sql.*; +import java.util.Map; +import java.util.concurrent.BlockingQueue; + +// TODO: This implementation deals with the JSON response. We can have respective implementations +public class BigQueryResultSetImpl extends AbstractJdbcResultSet + implements BigQueryResultSet { private final Schema schema; private final long totalRows; - private final T nextRow; + // private final ResultSet nextRow; + private final Map nameType; // TODO: Remove + private final BlockingQueue buffer; // TableRow + private T cursor; - public BigQueryResultSetImpl(Schema schema, long totalRows, T nextRow) { + // TODO : Implement a wrapper/struct like spanner + public BigQueryResultSetImpl( + Schema schema, long totalRows, Map nameType, BlockingQueue buffer) { this.schema = schema; this.totalRows = totalRows; - this.nextRow = nextRow; + this.nameType = nameType; + this.buffer = buffer; } @Override @@ -38,8 +52,123 @@ public long getTotalRows() { return totalRows; } + /* @Override + public ResultSet getNext() { + return null; + }*/ + + @Override + /*Advances the result set to the next row, returning false if no such row exists. Potentially blocking operation*/ + public boolean next() throws SQLException { + if (buffer.peek() == null) { // producer has no more rows left. + return false; + } + try { + cursor = buffer.take(); // advance the cursor,Potentially blocking operation + } catch (InterruptedException e) { + throw new SQLException("No rows left"); + } + + return true; + } + @Override - public T getNext() { + public Object getObject(String fieldName) throws SQLException { + if (cursor instanceof TableRow) { + TableRow currentRow = (TableRow) cursor; + if (currentRow == null) { + throw new SQLException("No rows left"); + } + return currentRow.get(fieldName); + } + // TODO: Add similar clauses for Apache Arrow return null; } + + @Override + public String getString(String fieldName) throws SQLException { + Object value = getObject(fieldName); + if (value == null) { + throw new SQLException("fieldName can't be null"); + } else if (!(value instanceof String)) { + throw new SQLException("value not in instance of String"); + } else { + return (String) value; + } + } + + @Override + public long getLong(String fieldName) throws SQLException { + Object value = getObject(fieldName); + if (value == null) { + throw new SQLException("fieldName can't be null"); + } else if (!(value instanceof Long)) { + throw new SQLException("value not in instance of Long"); + } else { + return Long.parseLong(String.valueOf(value)); + } + } + + @Override + public double getDouble(String fieldName) throws SQLException { + Object value = getObject(fieldName); + if (value == null) { + throw new SQLException("fieldName can't be null"); + } else if (!(value instanceof Double)) { + throw new SQLException("value not in instance of Double"); + } else { + return Double.parseDouble(String.valueOf(value)); + } + } + + @Override + public BigDecimal getBigDecimal(String fieldName) throws SQLException { + Object value = getObject(fieldName); + if (value == null) { + throw new SQLException("fieldName can't be null"); + } else if (!(value instanceof Long + || value instanceof Double + || value instanceof BigDecimal + || value instanceof String)) { + throw new SQLException("value cannot be converted to BigDecimal"); + } else { + return new BigDecimal(String.valueOf(value)); + } + } + + @Override + public boolean getBoolean(String fieldName) throws SQLException { + Object value = getObject(fieldName); + if (value == null) { + throw new SQLException("fieldName can't be null"); + } else if (!(value instanceof Boolean || (value instanceof String))) { + throw new SQLException("value not in instance of Boolean"); + } else { + return Boolean.parseBoolean(String.valueOf(value)); + } + } + + @Override + public byte getByte(String fieldName) throws SQLException { + Object value = getObject(fieldName); + if (value == null) { + throw new SQLException("fieldName can't be null"); + } else if (!(value instanceof Byte || (value instanceof String))) { + throw new SQLException("value not in instance of Boolean"); + } else { + return Byte.parseByte(String.valueOf(value)); + } + } + + @Override + public Timestamp getTimestamp(String fieldName) throws SQLException { + Object value = getObject(fieldName); + if (value == null) { + throw new SQLException("fieldName can't be null"); + } else if (!(value instanceof Long || value instanceof Timestamp || value instanceof String)) { + throw new SQLException("value cannot be converted to Timestamp"); + } else { + return Timestamp.valueOf(String.valueOf(value)); + } + } } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java index 30efe8e96..be6b62032 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java @@ -17,6 +17,7 @@ package com.google.cloud.bigquery; import com.google.api.services.bigquery.model.QueryParameter; +import java.sql.ResultSet; import java.util.List; import java.util.Map; @@ -39,23 +40,52 @@ public interface Connection { BigQueryDryRunResult dryRun(String sql) throws BigQuerySQLException; /** - * Execute a SQL statement that returns a single BigQueryResultSet + * Execute a SQL statement that returns a single ResultSet + * + *

Example of running a query. + * + * 

+   * {
+   *   @code
+   *   // ConnectionSettings connectionSettings =
+   *   //     ConnectionSettings.newBuilder()
+   *   //         .setRequestTimeout(10L)
+   *   //         .setMaxResults(100L)
+   *   //         .setUseQueryCache(true)
+   *   //         .build();
+   *   // Connection connection = bigquery.createConnection(connectionSettings);
+   *   String selectQuery = "SELECT corpus FROM `bigquery-public-data.samples.shakespeare` GROUP BY corpus;";
+   *   try (ResultSet rs = connection.executeSelect(selectQuery)) {
+   *       while (rs.next()) {
+   *           System.out.printf("%s,", rs.getString("corpus"));
+   *       }
+   *   } catch (BigQuerySQLException ex) {
+   *       // handle exception
+   *   }
+   * }
+   *
* * @param sql a static SQL SELECT statement - * @return a BigQueryResultSet that contains the data produced by the query + * @return a ResultSet that contains the data produced by the query * @exception BigQuerySQLException if a database access error occurs */ - BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException; + ResultSet executeSelect(String sql) throws BigQuerySQLException; /** - * Execute a SQL statement with query parameters that returns a single BigQueryResultSet + * Execute a SQL statement with query parameters that returns a single ResultSet * * @param sql typically a static SQL SELECT statement - * @param parameters named or positional parameters - * @return a BigQueryResultSet that contains the data produced by the query + * @param parameters named or positional parameters. The set of query parameters must either be + * all positional or all named parameters. Named parameters are denoted using an @ prefix, + * e.g. @myParam for a parameter named "myParam". + * @param labels the labels associated with this query. You can use these to organize and group + * your query jobs. Label keys and values can be no longer than 63 characters, can only + * contain lowercase letters, numeric characters, underscores and dashes. International + * characters are allowed. Label values are optional. Label keys must start with a letter and + * each label in the list must have a different key. + * @return a ResultSet that contains the data produced by the query * @exception BigQuerySQLException if a database access error occurs */ - BigQueryResultSet executeSelect( - String sql, List parameters, Map labels) + ResultSet executeSelect(String sql, List parameters, Map labels) throws BigQuerySQLException; } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 415fcb2c0..3f80c5c2d 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -16,31 +16,36 @@ package com.google.cloud.bigquery; -import com.google.api.services.bigquery.model.QueryParameter; -import com.google.api.services.bigquery.model.QueryRequest; -import com.google.api.services.bigquery.model.QueryResponse; +import static com.google.cloud.RetryHelper.runWithRetries; +import static java.net.HttpURLConnection.HTTP_NOT_FOUND; + +import com.google.api.services.bigquery.model.*; +import com.google.cloud.RetryHelper; import com.google.cloud.bigquery.spi.v2.BigQueryRpc; -import com.google.common.collect.Lists; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.Callable; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Maps; +import java.sql.ResultSet; +import java.util.*; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.stream.Collectors; /** Implementation for {@link Connection}, the generic BigQuery connection API (not JDBC). */ -final class ConnectionImpl implements Connection { +final class ConnectionImpl implements Connection { private ConnectionSettings connectionSettings; - private BigQueryOptions options; + private BigQueryOptions bigQueryOptions; private BigQueryRpc bigQueryRpc; private BigQueryRetryConfig retryConfig; ConnectionImpl( ConnectionSettings connectionSettings, - BigQueryOptions options, + BigQueryOptions bigQueryOptions, BigQueryRpc bigQueryRpc, BigQueryRetryConfig retryConfig) { this.connectionSettings = connectionSettings; - this.options = options; + this.bigQueryOptions = bigQueryOptions; this.bigQueryRpc = bigQueryRpc; this.retryConfig = retryConfig; } @@ -56,96 +61,248 @@ public BigQueryDryRunResult dryRun(String sql) throws BigQuerySQLException { } @Override - public BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException { - // use jobs.query if all properties of the Connection are supported + public ResultSet executeSelect(String sql) throws BigQuerySQLException { + // use jobs.query if all the properties of connectionSettings are supported if (isFastQuerySupported()) { - String projectId = options.getProjectId(); + String projectId = bigQueryOptions.getProjectId(); QueryRequest queryRequest = createQueryRequest(connectionSettings, sql, null, null); return queryRpc(projectId, queryRequest); } // use jobs.insert otherwise - return null; + com.google.api.services.bigquery.model.Job queryJob = + createQueryJob(sql, connectionSettings, null, null); + return null; // TODO getQueryResultsRpc(JobId.fromPb(queryJob.getJobReference())); } @Override - public BigQueryResultSet executeSelect( - String sql, List queryParameters, Map labels) + public ResultSet executeSelect( + String sql, List parameters, Map labels) throws BigQuerySQLException { + // use jobs.query if possible if (isFastQuerySupported()) { - final String projectId = options.getProjectId(); - final QueryRequest preparedQueryRequest = - createQueryRequest(connectionSettings, sql, queryParameters, labels); - // return queryRpc(projectId, preparedQueryRequest); + final String projectId = bigQueryOptions.getProjectId(); + final QueryRequest queryRequest = + createQueryRequest(connectionSettings, sql, parameters, labels); + return queryRpc(projectId, queryRequest); } - return null; + // use jobs.insert otherwise + com.google.api.services.bigquery.model.Job queryJob = + createQueryJob(sql, connectionSettings, parameters, labels); + return null; // TODO getQueryResultsRpc(JobId.fromPb(queryJob.getJobReference())); } - private BigQueryResultSet queryRpc(final String projectId, final QueryRequest queryRequest) { + private ResultSet queryRpc(final String projectId, final QueryRequest queryRequest) { com.google.api.services.bigquery.model.QueryResponse results; try { results = BigQueryRetryHelper.runWithRetries( - new Callable() { - @Override - public com.google.api.services.bigquery.model.QueryResponse call() { - return bigQueryRpc.queryRpc(projectId, queryRequest); - } - }, - options.getRetrySettings(), + () -> bigQueryRpc.queryRpc(projectId, queryRequest), + bigQueryOptions.getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, - options.getClock(), + bigQueryOptions.getClock(), retryConfig); } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); } if (results.getErrors() != null) { List bigQueryErrors = - Lists.transform(results.getErrors(), BigQueryError.FROM_PB_FUNCTION); + results.getErrors().stream() + .map(BigQueryError.FROM_PB_FUNCTION) + .collect(Collectors.toList()); // Throwing BigQueryException since there may be no JobId and we want to stay consistent // with the case where there there is a HTTP error throw new BigQueryException(bigQueryErrors); } - long numRows; - Schema schema; if (results.getJobComplete() && results.getSchema() != null) { - schema = Schema.fromPb(results.getSchema()); - numRows = results.getTotalRows().longValue(); + return processQueryResponseResults(results); } else { // Query is long running (> 10s) and hasn't completed yet, or query completed but didn't - // return the schema, fallback. Some operations don't return the schema and can be optimized - // here, but this is left as future work. + // return the schema, fallback to GetQueryResults. Some operations don't return the schema and + // can be optimized here, but this is left as future work. + long totalRows = results.getTotalRows().longValue(); + long pageRows = results.getRows().size(); JobId jobId = JobId.fromPb(results.getJobReference()); - return null; + return null; // TODO getQueryResultsWithJobId(totalRows, pageRows, null, jobId); } + } - // TODO(prasmish): WRITE A PRIVATE METHOD TO CACHE FIRST PAGE - // cacheFirstPage(...) - - if (results.getPageToken() != null) { - JobId jobId = JobId.fromPb(results.getJobReference()); - return new BigQueryResultSetImpl<>( - schema, numRows, null - /** iterate(cachedFirstPage)* */ - ); + // Get's field's name is to type mapping + private Map getFieldNameType(Schema schema) { + List fieldSchemas = schema.getFields().toPb(); + Map nameType = new HashMap<>(); + for (TableFieldSchema fieldSchema : fieldSchemas) { + nameType.put(fieldSchema.getName(), fieldSchema.getType()); } + return nameType; + } - // only 1 page of result - // TODO(prasmish): GET NEXT ROW AND PASS INTO BigQueryResultSetImpl + private ResultSet processQueryResponseResults( + com.google.api.services.bigquery.model.QueryResponse results) { + Schema schema; + long numRows; + schema = Schema.fromPb(results.getSchema()); + numRows = results.getTotalRows().longValue(); - // Query is long running (> 10s) and hasn't completed yet or has more than 1 pages of results - // Use tabledata.list or high through-put Read API to fetch the rest of the pages + // Producer thread for populating the buffer row by row + BlockingQueue buffer = + new LinkedBlockingDeque<>(1000); // TODO: Update the capacity. Prefetch limit + Runnable populateBufferRunnable = + () -> { // producer thread populating the buffer + List tableRows = results.getRows(); + for (TableRow tableRow : tableRows) { // TODO: Handle the logic of pagination + buffer.offer(tableRow); + } + if (results.getPageToken() == null) { + buffer.offer(null); // null marks the end of the records + } + }; + Thread populateBufferWorker = new Thread(populateBufferRunnable); + populateBufferWorker.start(); // child process to populate the buffer + + // only 1 page of result + if (results.getPageToken() == null) { + return new BigQueryResultSetImpl(schema, numRows, getFieldNameType(schema), buffer); + } + // use tabledata.list or Read API to fetch subsequent pages of results + long totalRows = results.getTotalRows().longValue(); + long pageRows = results.getRows().size(); JobId jobId = JobId.fromPb(results.getJobReference()); - if (useReadAPI(results)) { - // Discard the first page and use BigQuery Storage Read API to fetch the entire result set + return null; // TODO - Implement getQueryResultsWithJobId(totalRows, pageRows, schema, jobId); + } - } else { - // Use tabledata.list to fetch the rest of the result set + /* Returns query results using either tabledata.list or the high throughput Read API */ + private BigQueryResultSet getQueryResultsWithJobId( + long totalRows, long pageRows, Schema schema, JobId jobId) { + TableId destinationTable = queryJobsGetRpc(jobId); + return useReadAPI(totalRows, pageRows) + ? highThroughPutRead(destinationTable) + : tableDataListRpc(destinationTable, schema); + } + + /* Returns the destinationTable from jobId by calling jobs.get API */ + private TableId queryJobsGetRpc(JobId jobId) { + final JobId completeJobId = + jobId + .setProjectId(bigQueryOptions.getProjectId()) + .setLocation( + jobId.getLocation() == null && bigQueryOptions.getLocation() != null + ? bigQueryOptions.getLocation() + : jobId.getLocation()); + com.google.api.services.bigquery.model.Job jobPb; + try { + jobPb = + runWithRetries( + () -> + bigQueryRpc.getQueryJob( + completeJobId.getProject(), + completeJobId.getJob(), + completeJobId.getLocation()), + bigQueryOptions.getRetrySettings(), + BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, + bigQueryOptions.getClock()); + if (bigQueryOptions.getThrowNotFound() && jobPb == null) { + throw new BigQueryException(HTTP_NOT_FOUND, "Query job not found"); + } + } catch (RetryHelper.RetryHelperException e) { + throw BigQueryException.translateAndThrow(e); + } + Job job = Job.fromPb(bigQueryOptions.getService(), jobPb); + return ((QueryJobConfiguration) job.getConfiguration()).getDestinationTable(); + } + + private BigQueryResultSet tableDataListRpc(TableId destinationTable, Schema schema) { + try { + final TableId completeTableId = + destinationTable.setProjectId( + Strings.isNullOrEmpty(destinationTable.getProject()) + ? bigQueryOptions.getProjectId() + : destinationTable.getProject()); + TableDataList results = + runWithRetries( + () -> + bigQueryOptions + .getBigQueryRpcV2() + .listTableDataWithRowLimit( + completeTableId.getProject(), + completeTableId.getDataset(), + completeTableId.getTable(), + connectionSettings.getPrefetchedRowLimit()), + bigQueryOptions.getRetrySettings(), + BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, + bigQueryOptions.getClock()); + + long numRows = results.getTotalRows() == null ? 0 : results.getTotalRows().longValue(); + /* return new BigQueryResultSetImpl( + schema, numRows, null + */ + /* TODO: iterate(pagesAfterFirstPage) */ + /* );*/ + return null; // TODO: Plugin the buffer logic + } catch (RetryHelper.RetryHelperException e) { + throw BigQueryException.translateAndThrow(e); } + } + + private BigQueryResultSet highThroughPutRead(TableId destinationTable) { return null; } + /* Returns results of the query associated with the provided job using jobs.getQueryResults API */ + private BigQueryResultSet getQueryResultsRpc(JobId jobId) { + JobId completeJobId = + jobId + .setProjectId(bigQueryOptions.getProjectId()) + .setLocation( + jobId.getLocation() == null && bigQueryOptions.getLocation() != null + ? bigQueryOptions.getLocation() + : jobId.getLocation()); + try { + GetQueryResultsResponse results = + BigQueryRetryHelper.runWithRetries( + () -> + bigQueryRpc.getQueryResultsWithRowLimit( + completeJobId.getProject(), + completeJobId.getJob(), + completeJobId.getLocation(), + connectionSettings.getPrefetchedRowLimit()), + bigQueryOptions.getRetrySettings(), + BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, + bigQueryOptions.getClock(), + retryConfig); + + if (results.getErrors() != null) { + List bigQueryErrors = + results.getErrors().stream() + .map(BigQueryError.FROM_PB_FUNCTION) + .collect(Collectors.toList()); + // Throwing BigQueryException since there may be no JobId and we want to stay consistent + // with the case where there there is a HTTP error + throw new BigQueryException(bigQueryErrors); + } + return processGetQueryResults(jobId, results); + } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) { + throw BigQueryException.translateAndThrow(e); + } + } + + private BigQueryResultSet processGetQueryResults(JobId jobId, GetQueryResultsResponse results) { + long numRows = results.getTotalRows() == null ? 0 : results.getTotalRows().longValue(); + Schema schema = results.getSchema() == null ? null : Schema.fromPb(results.getSchema()); + + // only use this API for the first page of result + if (results.getPageToken() == null) { + // return new BigQueryResultSetImpl(schema, numRows, null /* TODO: iterate(cachedFirstPage) + // */); + return null; // TODO: Plugin the buffer logic + } + // use tabledata.list or Read API to fetch subsequent pages of results + long totalRows = results.getTotalRows() == null ? 0 : results.getTotalRows().longValue(); + long pageRows = results.getRows().size(); + return getQueryResultsWithJobId(totalRows, pageRows, schema, jobId); + } + private boolean isFastQuerySupported() { return connectionSettings.getClustering() == null && connectionSettings.getCreateDisposition() == null @@ -162,9 +319,7 @@ private boolean isFastQuerySupported() { && connectionSettings.getWriteDisposition() == null; } - private boolean useReadAPI(QueryResponse response) { - long totalRows = response.getTotalRows().longValue(); - long pageRows = response.getRows().size(); + private boolean useReadAPI(Long totalRows, Long pageRows) { long resultRatio = totalRows / pageRows; return resultRatio > connectionSettings @@ -183,8 +338,9 @@ private QueryRequest createQueryRequest( if (connectionSettings.getConnectionProperties() != null) { content.setConnectionProperties( - Lists.transform( - connectionSettings.getConnectionProperties(), ConnectionProperty.TO_PB_FUNCTION)); + connectionSettings.getConnectionProperties().stream() + .map(ConnectionProperty.TO_PB_FUNCTION) + .collect(Collectors.toList())); } if (connectionSettings.getDefaultDataset() != null) { content.setDefaultDataset(connectionSettings.getDefaultDataset().toPb()); @@ -205,4 +361,116 @@ private QueryRequest createQueryRequest( content.setRequestId(requestId); return content; } + + private com.google.api.services.bigquery.model.Job createQueryJob( + String sql, + ConnectionSettings connectionSettings, + List queryParameters, + Map labels) { + com.google.api.services.bigquery.model.JobConfiguration configurationPb = + new com.google.api.services.bigquery.model.JobConfiguration(); + JobConfigurationQuery queryConfigurationPb = new JobConfigurationQuery(); + queryConfigurationPb.setQuery(sql); + if (queryParameters != null) { + queryConfigurationPb.setQueryParameters(queryParameters); + if (queryParameters.get(0).getName() == null) { + queryConfigurationPb.setParameterMode("POSITIONAL"); + } else { + queryConfigurationPb.setParameterMode("NAMED"); + } + } + if (connectionSettings.getDestinationTable() != null) { + queryConfigurationPb.setDestinationTable(connectionSettings.getDestinationTable().toPb()); + } + if (connectionSettings.getTableDefinitions() != null) { + queryConfigurationPb.setTableDefinitions( + Maps.transformValues( + connectionSettings.getTableDefinitions(), + ExternalTableDefinition.TO_EXTERNAL_DATA_FUNCTION)); + } + if (connectionSettings.getUserDefinedFunctions() != null) { + queryConfigurationPb.setUserDefinedFunctionResources( + connectionSettings.getUserDefinedFunctions().stream() + .map(UserDefinedFunction.TO_PB_FUNCTION) + .collect(Collectors.toList())); + } + if (connectionSettings.getCreateDisposition() != null) { + queryConfigurationPb.setCreateDisposition( + connectionSettings.getCreateDisposition().toString()); + } + if (connectionSettings.getWriteDisposition() != null) { + queryConfigurationPb.setWriteDisposition(connectionSettings.getWriteDisposition().toString()); + } + if (connectionSettings.getDefaultDataset() != null) { + queryConfigurationPb.setDefaultDataset(connectionSettings.getDefaultDataset().toPb()); + } + if (connectionSettings.getPriority() != null) { + queryConfigurationPb.setPriority(connectionSettings.getPriority().toString()); + } + if (connectionSettings.getAllowLargeResults() != null) { + queryConfigurationPb.setAllowLargeResults(connectionSettings.getAllowLargeResults()); + } + if (connectionSettings.getUseQueryCache() != null) { + queryConfigurationPb.setUseQueryCache(connectionSettings.getUseQueryCache()); + } + if (connectionSettings.getFlattenResults() != null) { + queryConfigurationPb.setFlattenResults(connectionSettings.getFlattenResults()); + } + if (connectionSettings.getMaximumBillingTier() != null) { + queryConfigurationPb.setMaximumBillingTier(connectionSettings.getMaximumBillingTier()); + } + if (connectionSettings.getMaximumBytesBilled() != null) { + queryConfigurationPb.setMaximumBytesBilled(connectionSettings.getMaximumBytesBilled()); + } + if (connectionSettings.getSchemaUpdateOptions() != null) { + ImmutableList.Builder schemaUpdateOptionsBuilder = new ImmutableList.Builder<>(); + for (JobInfo.SchemaUpdateOption schemaUpdateOption : + connectionSettings.getSchemaUpdateOptions()) { + schemaUpdateOptionsBuilder.add(schemaUpdateOption.name()); + } + queryConfigurationPb.setSchemaUpdateOptions(schemaUpdateOptionsBuilder.build()); + } + if (connectionSettings.getDestinationEncryptionConfiguration() != null) { + queryConfigurationPb.setDestinationEncryptionConfiguration( + connectionSettings.getDestinationEncryptionConfiguration().toPb()); + } + if (connectionSettings.getTimePartitioning() != null) { + queryConfigurationPb.setTimePartitioning(connectionSettings.getTimePartitioning().toPb()); + } + if (connectionSettings.getClustering() != null) { + queryConfigurationPb.setClustering(connectionSettings.getClustering().toPb()); + } + if (connectionSettings.getRangePartitioning() != null) { + queryConfigurationPb.setRangePartitioning(connectionSettings.getRangePartitioning().toPb()); + } + if (connectionSettings.getConnectionProperties() != null) { + queryConfigurationPb.setConnectionProperties( + connectionSettings.getConnectionProperties().stream() + .map(ConnectionProperty.TO_PB_FUNCTION) + .collect(Collectors.toList())); + } + if (connectionSettings.getJobTimeoutMs() != null) { + configurationPb.setJobTimeoutMs(connectionSettings.getJobTimeoutMs()); + } + if (labels != null) { + configurationPb.setLabels(labels); + } + configurationPb.setQuery(queryConfigurationPb); + + com.google.api.services.bigquery.model.Job jobPb = + JobInfo.of(QueryJobConfiguration.fromPb(configurationPb)).toPb(); + com.google.api.services.bigquery.model.Job queryJob; + try { + queryJob = + BigQueryRetryHelper.runWithRetries( + () -> bigQueryRpc.createJobForQuery(jobPb), + bigQueryOptions.getRetrySettings(), + BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, + bigQueryOptions.getClock(), + retryConfig); + } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) { + throw BigQueryException.translateAndThrow(e); + } + return queryJob; + } } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java index 6fe0cbce9..1827312e6 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java @@ -46,15 +46,23 @@ public abstract class ConnectionSettings { /** Returns the maximum number of rows of data */ public abstract Long getMaxResults(); + /** Returns the number of rows of data to pre-fetch */ + public abstract Long getPrefetchedRowLimit(); + /** Returns whether to look for the result in the query cache */ public abstract Boolean getUseQueryCache(); + /** + * Returns whether nested and repeated fields should be flattened. If set to {@code false} {@link + * QueryJobConfiguration.Builder#setAllowLargeResults(Boolean)} must be {@code true}. + * + * @see Flatten + */ + public abstract Boolean getFlattenResults(); + /** Returns the BigQuery Storage read API configuration */ public abstract ReadClientConnectionConfiguration getReadClientConnectionConfiguration(); - /** Returns the return row serialization format */ - public abstract ResponseRowFormat getResponseRowFormat(); - /** * Below properties are only supported by jobs.insert API and not yet supported by jobs.query API * * @@ -90,6 +98,16 @@ public abstract class ConnectionSettings { /** Returns the query priority. */ public abstract Priority getPriority(); + /** + * Returns whether the job is enabled to create arbitrarily large results. If {@code true} the + * query is allowed to create large results at a slight cost in performance. the query is allowed + * to create large results at a slight cost in performance. + * + * @see Returning + * Large Query Results + */ + public abstract Boolean getAllowLargeResults(); + /** Returns the range partitioning specification for the table */ public abstract RangePartitioning getRangePartitioning(); @@ -183,6 +201,13 @@ public abstract static class Builder { */ public abstract Builder setMaxResults(Long maxResults); + /** + * Sets the number of rows of data to pre-fetch during query execution. + * + * @param prefetchedRowLimit prefetchedRowLimit or {@code null} for none + */ + public abstract Builder setPrefetchedRowLimit(Long prefetchedRowLimit); + /** * Sets whether to look for the result in the query cache. The query cache is a best-effort * cache that will be flushed whenever tables in the query are modified. Moreover, the query @@ -193,6 +218,15 @@ public abstract static class Builder { */ public abstract Builder setUseQueryCache(Boolean useQueryCache); + /** + * Sets whether nested and repeated fields should be flattened. If set to {@code false} {@link + * QueryJobConfiguration.Builder#setAllowLargeResults(Boolean)} must be {@code true}. By default + * results are flattened. + * + * @see Flatten + */ + public abstract Builder setFlattenResults(Boolean flattenResults); + /** * Sets the values necessary to determine whether table result will be read using the BigQuery * Storage client Read API. The BigQuery Storage client Read API will be used to read the query @@ -208,13 +242,6 @@ public abstract static class Builder { public abstract Builder setReadClientConnectionConfiguration( ReadClientConnectionConfiguration readClientConnectionConfiguration); - /** - * Sets the row format received from the BigQuery API. - * - * @param responseRowFormat or {@code null} for none - */ - public abstract Builder setResponseRowFormat(ResponseRowFormat responseRowFormat); - /** Sets the clustering specification for the destination table. */ public abstract Builder setClustering(Clustering clustering); @@ -265,6 +292,16 @@ public abstract Builder setDestinationEncryptionConfiguration( */ public abstract Builder setPriority(Priority priority); + /** + * Sets whether the job is enabled to create arbitrarily large results. If {@code true} the + * query is allowed to create large results at a slight cost in performance. If {@code true} + * {@link ConnectionSettings.Builder#setDestinationTable(TableId)} must be provided. + * + * @see Returning + * Large Query Results + */ + public abstract Builder setAllowLargeResults(Boolean allowLargeResults); + /** * Range partitioning specification for this table. Only one of timePartitioning and * rangePartitioning should be specified. diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ResponseRowFormat.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ResponseRowFormat.java deleted file mode 100644 index 3e03b63e3..000000000 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ResponseRowFormat.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.bigquery; - -/** Supported row serialization formats * */ -public enum ResponseRowFormat { - GenericRecord, - JSON -} diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java index 06488c5b4..f09befd18 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java @@ -122,6 +122,13 @@ Boolean getBoolean(Map options) { */ Job create(Job job, Map options); + /** + * Creates a new query job. + * + * @throws BigQueryException upon failure + */ + Job createJobForQuery(Job job); + /** * Delete the requested dataset. * @@ -246,6 +253,14 @@ TableDataInsertAllResponse insertAll( TableDataList listTableData( String projectId, String datasetId, String tableId, Map options); + /** + * Lists the table's rows with a limit on how many rows of data to pre-fetch. + * + * @throws BigQueryException upon failure + */ + TableDataList listTableDataWithRowLimit( + String projectId, String datasetId, String tableId, Long rowLimit); + /** * Returns the requested job or {@code null} if not found. * @@ -253,6 +268,13 @@ TableDataList listTableData( */ Job getJob(String projectId, String jobId, String location, Map options); + /** + * Returns the requested query job or {@code null} if not found. + * + * @throws BigQueryException upon failure + */ + Job getQueryJob(String projectId, String jobId, String location); + /** * Lists the project's jobs. * @@ -286,6 +308,15 @@ TableDataList listTableData( GetQueryResultsResponse getQueryResults( String projectId, String jobId, String location, Map options); + /** + * Returns results of the query with a limit on how many rows of data to pre-fetch associated with + * the provided job. + * + * @throws BigQueryException upon failure + */ + GetQueryResultsResponse getQueryResultsWithRowLimit( + String projectId, String jobId, String location, Long preFetchedRowLimit); + /** * Runs a BigQuery SQL query synchronously and returns query results if the query completes within * a specified timeout. diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java index 24d7dd6b0..620b9223e 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java @@ -221,6 +221,19 @@ public Job create(Job job, Map options) { } } + @Override + public Job createJobForQuery(Job job) { + try { + String projectId = + job.getJobReference() != null + ? job.getJobReference().getProjectId() + : this.options.getProjectId(); + return bigquery.jobs().insert(projectId, job).setPrettyPrint(false).execute(); + } catch (IOException ex) { + throw translate(ex); + } + } + @Override public boolean deleteDataset(String projectId, String datasetId, Map options) { try { @@ -515,6 +528,21 @@ public TableDataList listTableData( } } + @Override + public TableDataList listTableDataWithRowLimit( + String projectId, String datasetId, String tableId, Long preFetchedRowLimit) { + try { + return bigquery + .tabledata() + .list(projectId, datasetId, tableId) + .setPrettyPrint(false) + .setMaxResults(preFetchedRowLimit) + .execute(); + } catch (IOException ex) { + throw translate(ex); + } + } + @Override public Job getJob(String projectId, String jobId, String location, Map options) { try { @@ -534,6 +562,24 @@ public Job getJob(String projectId, String jobId, String location, Map> listJobs(String projectId, Map options) { try { @@ -644,6 +690,22 @@ public GetQueryResultsResponse getQueryResults( } } + @Override + public GetQueryResultsResponse getQueryResultsWithRowLimit( + String projectId, String jobId, String location, Long preFetchedRowLimit) { + try { + return bigquery + .jobs() + .getQueryResults(projectId, jobId) + .setPrettyPrint(false) + .setLocation(location) + .setMaxResults(preFetchedRowLimit) + .execute(); + } catch (IOException ex) { + throw translate(ex); + } + } + @Override public QueryResponse queryRpc(String projectId, QueryRequest content) { try { From 0a21d4e5ba4f4d8a373bc141bc9412e23bae01bf Mon Sep 17 00:00:00 2001 From: stephwang Date: Fri, 17 Sep 2021 15:12:38 -0400 Subject: [PATCH 004/277] clean up --- .../google/cloud/bigquery/BigQueryResultSetImpl.java | 12 ++++-------- .../com/google/cloud/bigquery/ConnectionImpl.java | 12 +----------- 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index faaacd835..636cb43c5 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -18,8 +18,8 @@ import com.google.api.services.bigquery.model.TableRow; import java.math.BigDecimal; -import java.sql.*; -import java.util.Map; +import java.sql.SQLException; +import java.sql.Timestamp; import java.util.concurrent.BlockingQueue; // TODO: This implementation deals with the JSON response. We can have respective implementations @@ -28,17 +28,13 @@ public class BigQueryResultSetImpl extends AbstractJdbcResultSet private final Schema schema; private final long totalRows; - // private final ResultSet nextRow; - private final Map nameType; // TODO: Remove - private final BlockingQueue buffer; // TableRow + private final BlockingQueue buffer; private T cursor; // TODO : Implement a wrapper/struct like spanner - public BigQueryResultSetImpl( - Schema schema, long totalRows, Map nameType, BlockingQueue buffer) { + public BigQueryResultSetImpl(Schema schema, long totalRows, BlockingQueue buffer) { this.schema = schema; this.totalRows = totalRows; - this.nameType = nameType; this.buffer = buffer; } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 3f80c5c2d..d51d02396 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -127,16 +127,6 @@ private ResultSet queryRpc(final String projectId, final QueryRequest queryReque } } - // Get's field's name is to type mapping - private Map getFieldNameType(Schema schema) { - List fieldSchemas = schema.getFields().toPb(); - Map nameType = new HashMap<>(); - for (TableFieldSchema fieldSchema : fieldSchemas) { - nameType.put(fieldSchema.getName(), fieldSchema.getType()); - } - return nameType; - } - private ResultSet processQueryResponseResults( com.google.api.services.bigquery.model.QueryResponse results) { Schema schema; @@ -162,7 +152,7 @@ private ResultSet processQueryResponseResults( // only 1 page of result if (results.getPageToken() == null) { - return new BigQueryResultSetImpl(schema, numRows, getFieldNameType(schema), buffer); + return new BigQueryResultSetImpl(schema, numRows, buffer); } // use tabledata.list or Read API to fetch subsequent pages of results long totalRows = results.getTotalRows().longValue(); From a566ae0bfc46122103b61e1134e7116c445322c7 Mon Sep 17 00:00:00 2001 From: stephwang Date: Thu, 23 Sep 2021 15:18:23 -0400 Subject: [PATCH 005/277] update javadoc --- .../src/main/java/com/google/cloud/bigquery/Connection.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java index be6b62032..c39382953 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java @@ -55,7 +55,8 @@ public interface Connection { * // .build(); * // Connection connection = bigquery.createConnection(connectionSettings); * String selectQuery = "SELECT corpus FROM `bigquery-public-data.samples.shakespeare` GROUP BY corpus;"; - * try (ResultSet rs = connection.executeSelect(selectQuery)) { + * try (BigQueryResultSet bqResultSet = connection.executeSelect(selectQuery)) { + * ResultSet rs = bqResultSet.getUnderlyingResultSet(); * while (rs.next()) { * System.out.printf("%s,", rs.getString("corpus")); * } From 0b42e62a2be347543e45714c9b5c34e231ad073b Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 24 Sep 2021 12:01:39 +0530 Subject: [PATCH 006/277] Removing getInt from AbstractJdbcResultSet --- .../java/com/google/cloud/bigquery/AbstractJdbcResultSet.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/AbstractJdbcResultSet.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/AbstractJdbcResultSet.java index 4456317f2..48205377f 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/AbstractJdbcResultSet.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/AbstractJdbcResultSet.java @@ -932,10 +932,6 @@ public short getShort(String columnLabel) throws SQLException { throw new SQLFeatureNotSupportedException(); } - @Override - public int getInt(String columnLabel) throws SQLException { - throw new SQLFeatureNotSupportedException(); - } @Override public byte[] getBytes(String columnLabel) throws SQLException { From 101b8c1ca389c4299170a2578d32ce4c72f4d36f Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 24 Sep 2021 12:09:52 +0530 Subject: [PATCH 007/277] Removing getInt from AbstractJdbcResultSet --- .../java/com/google/cloud/bigquery/AbstractJdbcResultSet.java | 1 - 1 file changed, 1 deletion(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/AbstractJdbcResultSet.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/AbstractJdbcResultSet.java index 48205377f..256bbecf8 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/AbstractJdbcResultSet.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/AbstractJdbcResultSet.java @@ -932,7 +932,6 @@ public short getShort(String columnLabel) throws SQLException { throw new SQLFeatureNotSupportedException(); } - @Override public byte[] getBytes(String columnLabel) throws SQLException { throw new SQLFeatureNotSupportedException(); From a1f1e01502fedd13de6779b820bff9c68d9c1aef Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 24 Sep 2021 12:10:16 +0530 Subject: [PATCH 008/277] Implemented getInt --- .../google/cloud/bigquery/BigQueryResultSetImpl.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index 636cb43c5..dcebca70c 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -167,4 +167,16 @@ public Timestamp getTimestamp(String fieldName) throws SQLException { return Timestamp.valueOf(String.valueOf(value)); } } + + @Override + public int getInt(String fieldName) throws SQLException { + Object value = getObject(fieldName); + if (value == null) { + throw new SQLException("fieldName can't be null"); + } else if (!(value instanceof Integer || value instanceof String)) { + throw new SQLException("value cannot be converted to int"); + } else { + return Integer.parseInt(String.valueOf(value)); + } + } } From 07016802fdd70e4cf1293ead3084be507a5aa3cb Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 24 Sep 2021 12:47:00 +0530 Subject: [PATCH 009/277] Added getResultSet method, removed unimplemented methods --- .../com/google/cloud/bigquery/BigQueryResultSet.java | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java index 7a3277c3a..e66c8d7a8 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java @@ -16,7 +16,7 @@ package com.google.cloud.bigquery; -import java.sql.SQLException; +import java.sql.ResultSet; public interface BigQueryResultSet { @@ -29,12 +29,6 @@ public interface BigQueryResultSet { */ long getTotalRows(); - /** Returns the next row. Null if there is no more rows left. */ - // ResultSet getNext(); - - /*Advances the result set to the next row, returning false if no such row exists. Potentially blocking operation*/ - boolean next() throws SQLException; - - /*Returns the value of a String field if the field exists, otherwise returns null*/ - String getString(String fieldName) throws SQLException; + /*Returns the The underlying ResultSet Implementation*/ + ResultSet getResultSet(); } From d10091b4545d6df8cd8ca01cebb6221afd896f1d Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 24 Sep 2021 12:50:10 +0530 Subject: [PATCH 010/277] Wrapped ResultSet Implementation and exposed getResultSet method --- .../cloud/bigquery/BigQueryResultSetImpl.java | 226 +++++++++--------- 1 file changed, 116 insertions(+), 110 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index dcebca70c..c13e988ec 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -18,24 +18,26 @@ import com.google.api.services.bigquery.model.TableRow; import java.math.BigDecimal; +import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Timestamp; import java.util.concurrent.BlockingQueue; // TODO: This implementation deals with the JSON response. We can have respective implementations -public class BigQueryResultSetImpl extends AbstractJdbcResultSet - implements BigQueryResultSet { +public class BigQueryResultSetImpl implements BigQueryResultSet { private final Schema schema; private final long totalRows; private final BlockingQueue buffer; private T cursor; + private ResultSetWrapper underlyingResultSet = null; // TODO : Implement a wrapper/struct like spanner public BigQueryResultSetImpl(Schema schema, long totalRows, BlockingQueue buffer) { this.schema = schema; this.totalRows = totalRows; this.buffer = buffer; + underlyingResultSet = new ResultSetWrapper(); } @Override @@ -48,135 +50,139 @@ public long getTotalRows() { return totalRows; } - /* @Override - public ResultSet getNext() { - return null; - }*/ - @Override - /*Advances the result set to the next row, returning false if no such row exists. Potentially blocking operation*/ - public boolean next() throws SQLException { - if (buffer.peek() == null) { // producer has no more rows left. - return false; - } - try { - cursor = buffer.take(); // advance the cursor,Potentially blocking operation - } catch (InterruptedException e) { - throw new SQLException("No rows left"); - } - - return true; + public ResultSet getResultSet() { + return underlyingResultSet; } - @Override - public Object getObject(String fieldName) throws SQLException { - if (cursor instanceof TableRow) { - TableRow currentRow = (TableRow) cursor; - if (currentRow == null) { + private class ResultSetWrapper extends AbstractJdbcResultSet { + @Override + /*Advances the result set to the next row, returning false if no such row exists. Potentially blocking operation*/ + public boolean next() throws SQLException { + if (buffer.peek() == null) { // producer has no more rows left. + return false; + } + try { + cursor = buffer.take(); // advance the cursor,Potentially blocking operation + } catch (InterruptedException e) { throw new SQLException("No rows left"); } - return currentRow.get(fieldName); + + return true; } - // TODO: Add similar clauses for Apache Arrow - return null; - } - @Override - public String getString(String fieldName) throws SQLException { - Object value = getObject(fieldName); - if (value == null) { - throw new SQLException("fieldName can't be null"); - } else if (!(value instanceof String)) { - throw new SQLException("value not in instance of String"); - } else { - return (String) value; + @Override + public Object getObject(String fieldName) throws SQLException { + if (cursor instanceof TableRow) { + TableRow currentRow = (TableRow) cursor; + if (currentRow == null) { + throw new SQLException("No rows left"); + } + return currentRow.get(fieldName); + } + // TODO: Add similar clauses for Apache Arrow + return null; } - } - @Override - public long getLong(String fieldName) throws SQLException { - Object value = getObject(fieldName); - if (value == null) { - throw new SQLException("fieldName can't be null"); - } else if (!(value instanceof Long)) { - throw new SQLException("value not in instance of Long"); - } else { - return Long.parseLong(String.valueOf(value)); + @Override + public String getString(String fieldName) throws SQLException { + Object value = getObject(fieldName); + if (value == null) { + throw new SQLException("fieldName can't be null"); + } else if (!(value instanceof String)) { + throw new SQLException("value not in instance of String"); + } else { + return (String) value; + } } - } - @Override - public double getDouble(String fieldName) throws SQLException { - Object value = getObject(fieldName); - if (value == null) { - throw new SQLException("fieldName can't be null"); - } else if (!(value instanceof Double)) { - throw new SQLException("value not in instance of Double"); - } else { - return Double.parseDouble(String.valueOf(value)); + @Override + public long getLong(String fieldName) throws SQLException { + Object value = getObject(fieldName); + if (value == null) { + throw new SQLException("fieldName can't be null"); + } else if (!(value instanceof Long)) { + throw new SQLException("value not in instance of Long"); + } else { + return Long.parseLong(String.valueOf(value)); + } } - } - @Override - public BigDecimal getBigDecimal(String fieldName) throws SQLException { - Object value = getObject(fieldName); - if (value == null) { - throw new SQLException("fieldName can't be null"); - } else if (!(value instanceof Long - || value instanceof Double - || value instanceof BigDecimal - || value instanceof String)) { - throw new SQLException("value cannot be converted to BigDecimal"); - } else { - return new BigDecimal(String.valueOf(value)); + @Override + public double getDouble(String fieldName) throws SQLException { + Object value = getObject(fieldName); + if (value == null) { + throw new SQLException("fieldName can't be null"); + } else if (!(value instanceof Double)) { + throw new SQLException("value not in instance of Double"); + } else { + return Double.parseDouble(String.valueOf(value)); + } } - } - @Override - public boolean getBoolean(String fieldName) throws SQLException { - Object value = getObject(fieldName); - if (value == null) { - throw new SQLException("fieldName can't be null"); - } else if (!(value instanceof Boolean || (value instanceof String))) { - throw new SQLException("value not in instance of Boolean"); - } else { - return Boolean.parseBoolean(String.valueOf(value)); + @Override + public BigDecimal getBigDecimal(String fieldName) throws SQLException { + Object value = getObject(fieldName); + if (value == null) { + throw new SQLException("fieldName can't be null"); + } else if (!(value instanceof Long + || value instanceof Double + || value instanceof BigDecimal + || value instanceof String)) { + throw new SQLException("value cannot be converted to BigDecimal"); + } else { + return new BigDecimal(String.valueOf(value)); + } } - } - @Override - public byte getByte(String fieldName) throws SQLException { - Object value = getObject(fieldName); - if (value == null) { - throw new SQLException("fieldName can't be null"); - } else if (!(value instanceof Byte || (value instanceof String))) { - throw new SQLException("value not in instance of Boolean"); - } else { - return Byte.parseByte(String.valueOf(value)); + @Override + public boolean getBoolean(String fieldName) throws SQLException { + Object value = getObject(fieldName); + if (value == null) { + throw new SQLException("fieldName can't be null"); + } else if (!(value instanceof Boolean || (value instanceof String))) { + throw new SQLException("value not in instance of Boolean"); + } else { + return Boolean.parseBoolean(String.valueOf(value)); + } } - } - @Override - public Timestamp getTimestamp(String fieldName) throws SQLException { - Object value = getObject(fieldName); - if (value == null) { - throw new SQLException("fieldName can't be null"); - } else if (!(value instanceof Long || value instanceof Timestamp || value instanceof String)) { - throw new SQLException("value cannot be converted to Timestamp"); - } else { - return Timestamp.valueOf(String.valueOf(value)); + @Override + public byte getByte(String fieldName) throws SQLException { + Object value = getObject(fieldName); + if (value == null) { + throw new SQLException("fieldName can't be null"); + } else if (!(value instanceof Byte || (value instanceof String))) { + throw new SQLException("value not in instance of Boolean"); + } else { + return Byte.parseByte(String.valueOf(value)); + } } - } - @Override - public int getInt(String fieldName) throws SQLException { - Object value = getObject(fieldName); - if (value == null) { - throw new SQLException("fieldName can't be null"); - } else if (!(value instanceof Integer || value instanceof String)) { - throw new SQLException("value cannot be converted to int"); - } else { - return Integer.parseInt(String.valueOf(value)); + @Override + public Timestamp getTimestamp(String fieldName) throws SQLException { + Object value = getObject(fieldName); + if (value == null) { + throw new SQLException("fieldName can't be null"); + } else if (!(value instanceof Long + || value instanceof Timestamp + || value instanceof String)) { + throw new SQLException("value cannot be converted to Timestamp"); + } else { + return Timestamp.valueOf(String.valueOf(value)); + } + } + + @Override + public int getInt(String fieldName) throws SQLException { + Object value = getObject(fieldName); + if (value == null) { + throw new SQLException("fieldName can't be null"); + } else if (!(value instanceof Integer || value instanceof String)) { + throw new SQLException("value cannot be converted to int"); + } else { + return Integer.parseInt(String.valueOf(value)); + } } } } From f6df413ad1d4418dda150195f16d022e655a5491 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 24 Sep 2021 12:50:39 +0530 Subject: [PATCH 011/277] Modified executeSelect to Return BigQueryDryRunResult --- .../src/main/java/com/google/cloud/bigquery/Connection.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java index c39382953..1d165e44c 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java @@ -17,7 +17,6 @@ package com.google.cloud.bigquery; import com.google.api.services.bigquery.model.QueryParameter; -import java.sql.ResultSet; import java.util.List; import java.util.Map; @@ -70,7 +69,7 @@ public interface Connection { * @return a ResultSet that contains the data produced by the query * @exception BigQuerySQLException if a database access error occurs */ - ResultSet executeSelect(String sql) throws BigQuerySQLException; + BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException; /** * Execute a SQL statement with query parameters that returns a single ResultSet @@ -87,6 +86,7 @@ public interface Connection { * @return a ResultSet that contains the data produced by the query * @exception BigQuerySQLException if a database access error occurs */ - ResultSet executeSelect(String sql, List parameters, Map labels) + BigQueryResultSet executeSelect( + String sql, List parameters, Map labels) throws BigQuerySQLException; } From 8ac341af7a79c7c968de48a0554e1d0e5927553a Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 24 Sep 2021 12:51:12 +0530 Subject: [PATCH 012/277] Modified executeSelect to Return BigQueryDryRunResult --- .../java/com/google/cloud/bigquery/ConnectionImpl.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index d51d02396..b750f6485 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -25,7 +25,6 @@ import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; -import java.sql.ResultSet; import java.util.*; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingDeque; @@ -61,7 +60,7 @@ public BigQueryDryRunResult dryRun(String sql) throws BigQuerySQLException { } @Override - public ResultSet executeSelect(String sql) throws BigQuerySQLException { + public BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException { // use jobs.query if all the properties of connectionSettings are supported if (isFastQuerySupported()) { String projectId = bigQueryOptions.getProjectId(); @@ -75,7 +74,7 @@ public ResultSet executeSelect(String sql) throws BigQuerySQLException { } @Override - public ResultSet executeSelect( + public BigQueryResultSet executeSelect( String sql, List parameters, Map labels) throws BigQuerySQLException { // use jobs.query if possible @@ -91,7 +90,7 @@ public ResultSet executeSelect( return null; // TODO getQueryResultsRpc(JobId.fromPb(queryJob.getJobReference())); } - private ResultSet queryRpc(final String projectId, final QueryRequest queryRequest) { + private BigQueryResultSet queryRpc(final String projectId, final QueryRequest queryRequest) { com.google.api.services.bigquery.model.QueryResponse results; try { results = @@ -127,7 +126,7 @@ private ResultSet queryRpc(final String projectId, final QueryRequest queryReque } } - private ResultSet processQueryResponseResults( + private BigQueryResultSet processQueryResponseResults( com.google.api.services.bigquery.model.QueryResponse results) { Schema schema; long numRows; From abdababb5727b6ce360f5cb33af63a8ad74b0646 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 24 Sep 2021 13:01:04 +0530 Subject: [PATCH 013/277] Modified executeSelect to Return BigQueryDryRunResult --- .../java/com/google/cloud/bigquery/BigQueryResultSetImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index c13e988ec..b91ed424c 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -37,7 +37,7 @@ public BigQueryResultSetImpl(Schema schema, long totalRows, BlockingQueue buf this.schema = schema; this.totalRows = totalRows; this.buffer = buffer; - underlyingResultSet = new ResultSetWrapper(); + this.underlyingResultSet = new ResultSetWrapper(); } @Override From 4584ef2ebfa6ac87ccab12cf051e31471084668f Mon Sep 17 00:00:00 2001 From: stephwang Date: Fri, 17 Sep 2021 15:12:38 -0400 Subject: [PATCH 014/277] update BigQueryResultSet to return wrapped ResultSet update javadoc Removing getInt from AbstractJdbcResultSet Removing getInt from AbstractJdbcResultSet Implemented getInt Added getResultSet method, removed unimplemented methods Wrapped ResultSet Implementation and exposed getResultSet method Modified executeSelect to Return BigQueryDryRunResult Modified executeSelect to Return BigQueryDryRunResult Modified executeSelect to Return BigQueryDryRunResult update javadoc --- .../cloud/bigquery/AbstractJdbcResultSet.java | 5 - .../cloud/bigquery/BigQueryResultSet.java | 12 +- .../cloud/bigquery/BigQueryResultSetImpl.java | 230 ++++++++++-------- .../com/google/cloud/bigquery/Connection.java | 9 +- .../google/cloud/bigquery/ConnectionImpl.java | 21 +- 5 files changed, 135 insertions(+), 142 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/AbstractJdbcResultSet.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/AbstractJdbcResultSet.java index 4456317f2..256bbecf8 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/AbstractJdbcResultSet.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/AbstractJdbcResultSet.java @@ -932,11 +932,6 @@ public short getShort(String columnLabel) throws SQLException { throw new SQLFeatureNotSupportedException(); } - @Override - public int getInt(String columnLabel) throws SQLException { - throw new SQLFeatureNotSupportedException(); - } - @Override public byte[] getBytes(String columnLabel) throws SQLException { throw new SQLFeatureNotSupportedException(); diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java index 7a3277c3a..e66c8d7a8 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java @@ -16,7 +16,7 @@ package com.google.cloud.bigquery; -import java.sql.SQLException; +import java.sql.ResultSet; public interface BigQueryResultSet { @@ -29,12 +29,6 @@ public interface BigQueryResultSet { */ long getTotalRows(); - /** Returns the next row. Null if there is no more rows left. */ - // ResultSet getNext(); - - /*Advances the result set to the next row, returning false if no such row exists. Potentially blocking operation*/ - boolean next() throws SQLException; - - /*Returns the value of a String field if the field exists, otherwise returns null*/ - String getString(String fieldName) throws SQLException; + /*Returns the The underlying ResultSet Implementation*/ + ResultSet getResultSet(); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index faaacd835..b91ed424c 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -18,28 +18,26 @@ import com.google.api.services.bigquery.model.TableRow; import java.math.BigDecimal; -import java.sql.*; -import java.util.Map; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; import java.util.concurrent.BlockingQueue; // TODO: This implementation deals with the JSON response. We can have respective implementations -public class BigQueryResultSetImpl extends AbstractJdbcResultSet - implements BigQueryResultSet { +public class BigQueryResultSetImpl implements BigQueryResultSet { private final Schema schema; private final long totalRows; - // private final ResultSet nextRow; - private final Map nameType; // TODO: Remove - private final BlockingQueue buffer; // TableRow + private final BlockingQueue buffer; private T cursor; + private ResultSetWrapper underlyingResultSet = null; // TODO : Implement a wrapper/struct like spanner - public BigQueryResultSetImpl( - Schema schema, long totalRows, Map nameType, BlockingQueue buffer) { + public BigQueryResultSetImpl(Schema schema, long totalRows, BlockingQueue buffer) { this.schema = schema; this.totalRows = totalRows; - this.nameType = nameType; this.buffer = buffer; + this.underlyingResultSet = new ResultSetWrapper(); } @Override @@ -52,123 +50,139 @@ public long getTotalRows() { return totalRows; } - /* @Override - public ResultSet getNext() { - return null; - }*/ - @Override - /*Advances the result set to the next row, returning false if no such row exists. Potentially blocking operation*/ - public boolean next() throws SQLException { - if (buffer.peek() == null) { // producer has no more rows left. - return false; - } - try { - cursor = buffer.take(); // advance the cursor,Potentially blocking operation - } catch (InterruptedException e) { - throw new SQLException("No rows left"); - } - - return true; + public ResultSet getResultSet() { + return underlyingResultSet; } - @Override - public Object getObject(String fieldName) throws SQLException { - if (cursor instanceof TableRow) { - TableRow currentRow = (TableRow) cursor; - if (currentRow == null) { + private class ResultSetWrapper extends AbstractJdbcResultSet { + @Override + /*Advances the result set to the next row, returning false if no such row exists. Potentially blocking operation*/ + public boolean next() throws SQLException { + if (buffer.peek() == null) { // producer has no more rows left. + return false; + } + try { + cursor = buffer.take(); // advance the cursor,Potentially blocking operation + } catch (InterruptedException e) { throw new SQLException("No rows left"); } - return currentRow.get(fieldName); + + return true; } - // TODO: Add similar clauses for Apache Arrow - return null; - } - @Override - public String getString(String fieldName) throws SQLException { - Object value = getObject(fieldName); - if (value == null) { - throw new SQLException("fieldName can't be null"); - } else if (!(value instanceof String)) { - throw new SQLException("value not in instance of String"); - } else { - return (String) value; + @Override + public Object getObject(String fieldName) throws SQLException { + if (cursor instanceof TableRow) { + TableRow currentRow = (TableRow) cursor; + if (currentRow == null) { + throw new SQLException("No rows left"); + } + return currentRow.get(fieldName); + } + // TODO: Add similar clauses for Apache Arrow + return null; } - } - @Override - public long getLong(String fieldName) throws SQLException { - Object value = getObject(fieldName); - if (value == null) { - throw new SQLException("fieldName can't be null"); - } else if (!(value instanceof Long)) { - throw new SQLException("value not in instance of Long"); - } else { - return Long.parseLong(String.valueOf(value)); + @Override + public String getString(String fieldName) throws SQLException { + Object value = getObject(fieldName); + if (value == null) { + throw new SQLException("fieldName can't be null"); + } else if (!(value instanceof String)) { + throw new SQLException("value not in instance of String"); + } else { + return (String) value; + } } - } - @Override - public double getDouble(String fieldName) throws SQLException { - Object value = getObject(fieldName); - if (value == null) { - throw new SQLException("fieldName can't be null"); - } else if (!(value instanceof Double)) { - throw new SQLException("value not in instance of Double"); - } else { - return Double.parseDouble(String.valueOf(value)); + @Override + public long getLong(String fieldName) throws SQLException { + Object value = getObject(fieldName); + if (value == null) { + throw new SQLException("fieldName can't be null"); + } else if (!(value instanceof Long)) { + throw new SQLException("value not in instance of Long"); + } else { + return Long.parseLong(String.valueOf(value)); + } } - } - @Override - public BigDecimal getBigDecimal(String fieldName) throws SQLException { - Object value = getObject(fieldName); - if (value == null) { - throw new SQLException("fieldName can't be null"); - } else if (!(value instanceof Long - || value instanceof Double - || value instanceof BigDecimal - || value instanceof String)) { - throw new SQLException("value cannot be converted to BigDecimal"); - } else { - return new BigDecimal(String.valueOf(value)); + @Override + public double getDouble(String fieldName) throws SQLException { + Object value = getObject(fieldName); + if (value == null) { + throw new SQLException("fieldName can't be null"); + } else if (!(value instanceof Double)) { + throw new SQLException("value not in instance of Double"); + } else { + return Double.parseDouble(String.valueOf(value)); + } } - } - @Override - public boolean getBoolean(String fieldName) throws SQLException { - Object value = getObject(fieldName); - if (value == null) { - throw new SQLException("fieldName can't be null"); - } else if (!(value instanceof Boolean || (value instanceof String))) { - throw new SQLException("value not in instance of Boolean"); - } else { - return Boolean.parseBoolean(String.valueOf(value)); + @Override + public BigDecimal getBigDecimal(String fieldName) throws SQLException { + Object value = getObject(fieldName); + if (value == null) { + throw new SQLException("fieldName can't be null"); + } else if (!(value instanceof Long + || value instanceof Double + || value instanceof BigDecimal + || value instanceof String)) { + throw new SQLException("value cannot be converted to BigDecimal"); + } else { + return new BigDecimal(String.valueOf(value)); + } } - } - @Override - public byte getByte(String fieldName) throws SQLException { - Object value = getObject(fieldName); - if (value == null) { - throw new SQLException("fieldName can't be null"); - } else if (!(value instanceof Byte || (value instanceof String))) { - throw new SQLException("value not in instance of Boolean"); - } else { - return Byte.parseByte(String.valueOf(value)); + @Override + public boolean getBoolean(String fieldName) throws SQLException { + Object value = getObject(fieldName); + if (value == null) { + throw new SQLException("fieldName can't be null"); + } else if (!(value instanceof Boolean || (value instanceof String))) { + throw new SQLException("value not in instance of Boolean"); + } else { + return Boolean.parseBoolean(String.valueOf(value)); + } } - } - @Override - public Timestamp getTimestamp(String fieldName) throws SQLException { - Object value = getObject(fieldName); - if (value == null) { - throw new SQLException("fieldName can't be null"); - } else if (!(value instanceof Long || value instanceof Timestamp || value instanceof String)) { - throw new SQLException("value cannot be converted to Timestamp"); - } else { - return Timestamp.valueOf(String.valueOf(value)); + @Override + public byte getByte(String fieldName) throws SQLException { + Object value = getObject(fieldName); + if (value == null) { + throw new SQLException("fieldName can't be null"); + } else if (!(value instanceof Byte || (value instanceof String))) { + throw new SQLException("value not in instance of Boolean"); + } else { + return Byte.parseByte(String.valueOf(value)); + } + } + + @Override + public Timestamp getTimestamp(String fieldName) throws SQLException { + Object value = getObject(fieldName); + if (value == null) { + throw new SQLException("fieldName can't be null"); + } else if (!(value instanceof Long + || value instanceof Timestamp + || value instanceof String)) { + throw new SQLException("value cannot be converted to Timestamp"); + } else { + return Timestamp.valueOf(String.valueOf(value)); + } + } + + @Override + public int getInt(String fieldName) throws SQLException { + Object value = getObject(fieldName); + if (value == null) { + throw new SQLException("fieldName can't be null"); + } else if (!(value instanceof Integer || value instanceof String)) { + throw new SQLException("value cannot be converted to int"); + } else { + return Integer.parseInt(String.valueOf(value)); + } } } } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java index be6b62032..f254cadc0 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java @@ -17,7 +17,6 @@ package com.google.cloud.bigquery; import com.google.api.services.bigquery.model.QueryParameter; -import java.sql.ResultSet; import java.util.List; import java.util.Map; @@ -55,7 +54,8 @@ public interface Connection { * // .build(); * // Connection connection = bigquery.createConnection(connectionSettings); * String selectQuery = "SELECT corpus FROM `bigquery-public-data.samples.shakespeare` GROUP BY corpus;"; - * try (ResultSet rs = connection.executeSelect(selectQuery)) { + * try (BigQueryResultSet bqResultSet = connection.executeSelect(selectQuery)) { + * ResultSet rs = bqResultSet.getResultSet(); * while (rs.next()) { * System.out.printf("%s,", rs.getString("corpus")); * } @@ -69,7 +69,7 @@ public interface Connection { * @return a ResultSet that contains the data produced by the query * @exception BigQuerySQLException if a database access error occurs */ - ResultSet executeSelect(String sql) throws BigQuerySQLException; + BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException; /** * Execute a SQL statement with query parameters that returns a single ResultSet @@ -86,6 +86,7 @@ public interface Connection { * @return a ResultSet that contains the data produced by the query * @exception BigQuerySQLException if a database access error occurs */ - ResultSet executeSelect(String sql, List parameters, Map labels) + BigQueryResultSet executeSelect( + String sql, List parameters, Map labels) throws BigQuerySQLException; } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 3f80c5c2d..b750f6485 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -25,7 +25,6 @@ import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; -import java.sql.ResultSet; import java.util.*; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingDeque; @@ -61,7 +60,7 @@ public BigQueryDryRunResult dryRun(String sql) throws BigQuerySQLException { } @Override - public ResultSet executeSelect(String sql) throws BigQuerySQLException { + public BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException { // use jobs.query if all the properties of connectionSettings are supported if (isFastQuerySupported()) { String projectId = bigQueryOptions.getProjectId(); @@ -75,7 +74,7 @@ public ResultSet executeSelect(String sql) throws BigQuerySQLException { } @Override - public ResultSet executeSelect( + public BigQueryResultSet executeSelect( String sql, List parameters, Map labels) throws BigQuerySQLException { // use jobs.query if possible @@ -91,7 +90,7 @@ public ResultSet executeSelect( return null; // TODO getQueryResultsRpc(JobId.fromPb(queryJob.getJobReference())); } - private ResultSet queryRpc(final String projectId, final QueryRequest queryRequest) { + private BigQueryResultSet queryRpc(final String projectId, final QueryRequest queryRequest) { com.google.api.services.bigquery.model.QueryResponse results; try { results = @@ -127,17 +126,7 @@ private ResultSet queryRpc(final String projectId, final QueryRequest queryReque } } - // Get's field's name is to type mapping - private Map getFieldNameType(Schema schema) { - List fieldSchemas = schema.getFields().toPb(); - Map nameType = new HashMap<>(); - for (TableFieldSchema fieldSchema : fieldSchemas) { - nameType.put(fieldSchema.getName(), fieldSchema.getType()); - } - return nameType; - } - - private ResultSet processQueryResponseResults( + private BigQueryResultSet processQueryResponseResults( com.google.api.services.bigquery.model.QueryResponse results) { Schema schema; long numRows; @@ -162,7 +151,7 @@ private ResultSet processQueryResponseResults( // only 1 page of result if (results.getPageToken() == null) { - return new BigQueryResultSetImpl(schema, numRows, getFieldNameType(schema), buffer); + return new BigQueryResultSetImpl(schema, numRows, buffer); } // use tabledata.list or Read API to fetch subsequent pages of results long totalRows = results.getTotalRows().longValue(); From 1cb5596ac1fcca8007d0944adde5b599e288a5b3 Mon Sep 17 00:00:00 2001 From: stephwang Date: Mon, 27 Sep 2021 11:36:47 -0400 Subject: [PATCH 015/277] add test case and update ConnectionSettings annotation --- .../cloud/bigquery/ConnectionSettings.java | 23 +++++++++++++++++++ .../cloud/bigquery/it/ITBigQueryTest.java | 20 ++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java index 1827312e6..3d4480541 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java @@ -23,6 +23,7 @@ import com.google.cloud.bigquery.QueryJobConfiguration.Priority; import java.util.List; import java.util.Map; +import javax.annotation.Nullable; /** ConnectionSettings for setting up a BigQuery query connection. */ @AutoValue @@ -32,24 +33,30 @@ public abstract class ConnectionSettings { } /** Returns the synchronous response timeoutMs associated with this query */ + @Nullable public abstract Long getRequestTimeout(); /** Returns the connection properties for connection string with this query */ + @Nullable public abstract List getConnectionProperties(); /** Returns the default dataset */ public abstract DatasetId getDefaultDataset(); /** Returns the limits the bytes billed for this job */ + @Nullable public abstract Long getMaximumBytesBilled(); /** Returns the maximum number of rows of data */ + @Nullable public abstract Long getMaxResults(); /** Returns the number of rows of data to pre-fetch */ + @Nullable public abstract Long getPrefetchedRowLimit(); /** Returns whether to look for the result in the query cache */ + @Nullable public abstract Boolean getUseQueryCache(); /** @@ -58,9 +65,11 @@ public abstract class ConnectionSettings { * * @see Flatten */ + @Nullable public abstract Boolean getFlattenResults(); /** Returns the BigQuery Storage read API configuration */ + @Nullable public abstract ReadClientConnectionConfiguration getReadClientConnectionConfiguration(); /** @@ -69,6 +78,7 @@ public abstract class ConnectionSettings { */ /** Returns the clustering specification for the destination table. */ + @Nullable public abstract Clustering getClustering(); /** @@ -78,24 +88,30 @@ public abstract class ConnectionSettings { * href="https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.query.createDisposition"> * Create Disposition */ + @Nullable public abstract CreateDisposition getCreateDisposition(); /** Returns the custom encryption configuration (e.g., Cloud KMS keys) */ + @Nullable public abstract EncryptionConfiguration getDestinationEncryptionConfiguration(); /** * Returns the table where to put query results. If not provided a new table is created. This * value is required if {@link #allowLargeResults()} is {@code true}. */ + @Nullable public abstract TableId getDestinationTable(); /** Returns the timeout associated with this job */ + @Nullable public abstract Long getJobTimeoutMs(); /** Returns the optional billing tier limit for this job. */ + @Nullable public abstract Integer getMaximumBillingTier(); /** Returns the query priority. */ + @Nullable public abstract Priority getPriority(); /** @@ -106,9 +122,11 @@ public abstract class ConnectionSettings { * @see Returning * Large Query Results */ + @Nullable public abstract Boolean getAllowLargeResults(); /** Returns the range partitioning specification for the table */ + @Nullable public abstract RangePartitioning getRangePartitioning(); /** @@ -118,6 +136,7 @@ public abstract class ConnectionSettings { * table is a partition of a table, specified by partition decorators. For normal tables, * WRITE_TRUNCATE will always overwrite the schema. */ + @Nullable public abstract List getSchemaUpdateOptions(); /** @@ -126,9 +145,11 @@ public abstract class ConnectionSettings { * defining these properties, the data sources can be queried as if they were standard BigQuery * tables. */ + @Nullable public abstract Map getTableDefinitions(); /** Returns the time partitioning specification for the destination table. */ + @Nullable public abstract TimePartitioning getTimePartitioning(); /** @@ -136,6 +157,7 @@ public abstract class ConnectionSettings { * either be defined inline ({@link UserDefinedFunction.Type#INLINE}) or loaded from a Google * Cloud Storage URI ({@link UserDefinedFunction.Type#FROM_URI}. */ + @Nullable public abstract List getUserDefinedFunctions(); /** @@ -145,6 +167,7 @@ public abstract class ConnectionSettings { * href="https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.query.writeDisposition"> * Write Disposition */ + @Nullable public abstract WriteDisposition getWriteDisposition(); /** Returns a builder pre-populated using the current values of this field. */ diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 484e4c0f5..5a24682b4 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -50,8 +50,11 @@ import com.google.cloud.bigquery.BigQuery.TableOption; import com.google.cloud.bigquery.BigQueryError; import com.google.cloud.bigquery.BigQueryException; +import com.google.cloud.bigquery.BigQueryResultSet; import com.google.cloud.bigquery.Clustering; +import com.google.cloud.bigquery.Connection; import com.google.cloud.bigquery.ConnectionProperty; +import com.google.cloud.bigquery.ConnectionSettings; import com.google.cloud.bigquery.CopyJobConfiguration; import com.google.cloud.bigquery.Dataset; import com.google.cloud.bigquery.DatasetId; @@ -127,6 +130,8 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.nio.file.FileSystems; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -1951,6 +1956,21 @@ public void testQueryExternalHivePartitioningOptionCustomLayout() throws Interru assertTrue(bigquery.delete(tableId)); } + @Test + public void testExecuteQuerySinglePageTableRow() throws SQLException { + String query = "SELECT * FROM " + TABLE_ID_FASTQUERY.getTable(); + ConnectionSettings connectionSettings = + ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); + Connection connection = bigquery.createConnection(connectionSettings); + BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); + ResultSet rs = bigQueryResultSet.getResultSet(); + while (rs.next()) { + // System.out.println(rs.getString("TimestampField")); + } + // assertEquals(QUERY_RESULT_SCHEMA, bigQueryResultSet.getSchema()); + // assertEquals(2, bigQueryResultSet.getTotalRows()); + } + @Test public void testFastQueryMultipleRuns() throws InterruptedException { String query = From 4b80e1b2f621f7a336d0d6efb05f28e5fae6b09e Mon Sep 17 00:00:00 2001 From: stephwang Date: Mon, 27 Sep 2021 15:00:27 -0400 Subject: [PATCH 016/277] add unit test for ConnectiontSettings; minor refactoring --- .../google/cloud/bigquery/ConnectionImpl.java | 4 +- .../cloud/bigquery/ConnectionSettings.java | 6 +- .../ReadClientConnectionConfiguration.java | 4 + .../bigquery/ConnectionSettingsTest.java | 179 ++++++++++++++++++ 4 files changed, 188 insertions(+), 5 deletions(-) create mode 100644 google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionSettingsTest.java diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index b750f6485..91769b6e4 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -217,7 +217,7 @@ private BigQueryResultSet tableDataListRpc(TableId destinationTable, Schema sche completeTableId.getProject(), completeTableId.getDataset(), completeTableId.getTable(), - connectionSettings.getPrefetchedRowLimit()), + connectionSettings.getNumBufferedRows()), bigQueryOptions.getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, bigQueryOptions.getClock()); @@ -255,7 +255,7 @@ private BigQueryResultSet getQueryResultsRpc(JobId jobId) { completeJobId.getProject(), completeJobId.getJob(), completeJobId.getLocation(), - connectionSettings.getPrefetchedRowLimit()), + connectionSettings.getNumBufferedRows()), bigQueryOptions.getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, bigQueryOptions.getClock(), diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java index 3d4480541..06349080e 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java @@ -53,7 +53,7 @@ public abstract class ConnectionSettings { /** Returns the number of rows of data to pre-fetch */ @Nullable - public abstract Long getPrefetchedRowLimit(); + public abstract Long getNumBufferedRows(); /** Returns whether to look for the result in the query cache */ @Nullable @@ -227,9 +227,9 @@ public abstract static class Builder { /** * Sets the number of rows of data to pre-fetch during query execution. * - * @param prefetchedRowLimit prefetchedRowLimit or {@code null} for none + * @param numBufferedRows prefetchedRowLimit or {@code null} for none */ - public abstract Builder setPrefetchedRowLimit(Long prefetchedRowLimit); + public abstract Builder setNumBufferedRows(Long numBufferedRows); /** * Sets whether to look for the result in the query cache. The query cache is a best-effort diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ReadClientConnectionConfiguration.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ReadClientConnectionConfiguration.java index 7b683b565..96faa4802 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ReadClientConnectionConfiguration.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ReadClientConnectionConfiguration.java @@ -18,6 +18,7 @@ import com.google.auto.value.AutoValue; import java.io.Serializable; +import javax.annotation.Nullable; /** Represents BigQueryStorage Read client connection information. */ @AutoValue @@ -30,15 +31,18 @@ public abstract static class Builder { * Sets the total row count to page row count ratio used to determine whether to us the * BigQueryStorage Read client to fetch result sets after the first page. */ + @Nullable public abstract Builder setTotalToPageRowCountRatio(Long ratio); /** * Sets the minimum number of table rows in the query results used to determine whether to us * the BigQueryStorage Read client to fetch result sets after the first page. */ + @Nullable public abstract Builder setMinResultSize(Long numRows); /** Sets the buffer size during streaming from the BigQueryStorage Read client. */ + @Nullable public abstract Builder setBufferSize(Long bufferSize); /** Creates a {@code ReadClientConnectionConfiguration} object. */ diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionSettingsTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionSettingsTest.java new file mode 100644 index 000000000..b45a11bb2 --- /dev/null +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionSettingsTest.java @@ -0,0 +1,179 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import static org.junit.Assert.assertEquals; + +import com.google.cloud.bigquery.JobInfo.CreateDisposition; +import com.google.cloud.bigquery.JobInfo.SchemaUpdateOption; +import com.google.cloud.bigquery.JobInfo.WriteDisposition; +import com.google.cloud.bigquery.QueryJobConfiguration.Priority; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import java.util.List; +import java.util.Map; +import org.junit.Test; + +public class ConnectionSettingsTest { + private static final String TEST_PROJECT_ID = "test-project-id"; + private static final DatasetId DATASET_ID = DatasetId.of("dataset"); + private static final TableId TABLE_ID = TableId.of("dataset", "table"); + private static final Long REQUEST_TIMEOUT = 10l; + private static final Long NUM_BUFFERED_ROWS = 100l; + private static final Long MAX_RESULTS = 1000l; + private static final ReadClientConnectionConfiguration READ_CLIENT_CONNECTION_CONFIGURATION = + ReadClientConnectionConfiguration.newBuilder() + .setTotalToPageRowCountRatio(3l) + .setMinResultSize(100l) + .setBufferSize(100l) + .build(); + private static final List SOURCE_URIS = ImmutableList.of("uri1", "uri2"); + private static final String KEY = "time_zone"; + private static final String VALUE = "US/Eastern"; + private static final ConnectionProperty CONNECTION_PROPERTY = + ConnectionProperty.newBuilder().setKey(KEY).setValue(VALUE).build(); + private static final List CONNECTION_PROPERTIES = + ImmutableList.of(CONNECTION_PROPERTY); + private static final Field FIELD_SCHEMA1 = + Field.newBuilder("StringField", StandardSQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .setDescription("FieldDescription1") + .build(); + private static final Field FIELD_SCHEMA2 = + Field.newBuilder("IntegerField", StandardSQLTypeName.INT64) + .setMode(Field.Mode.REPEATED) + .setDescription("FieldDescription2") + .build(); + private static final Schema TABLE_SCHEMA = Schema.of(FIELD_SCHEMA1, FIELD_SCHEMA2); + private static final Integer MAX_BAD_RECORDS = 42; + private static final Boolean IGNORE_UNKNOWN_VALUES = true; + private static final String COMPRESSION = "GZIP"; + private static final CsvOptions CSV_OPTIONS = CsvOptions.newBuilder().build(); + private static final ExternalTableDefinition TABLE_CONFIGURATION = + ExternalTableDefinition.newBuilder(SOURCE_URIS, TABLE_SCHEMA, CSV_OPTIONS) + .setCompression(COMPRESSION) + .setIgnoreUnknownValues(IGNORE_UNKNOWN_VALUES) + .setMaxBadRecords(MAX_BAD_RECORDS) + .build(); + private static final Map TABLE_DEFINITIONS = + ImmutableMap.of("tableName", TABLE_CONFIGURATION); + private static final CreateDisposition CREATE_DISPOSITION = CreateDisposition.CREATE_IF_NEEDED; + private static final WriteDisposition WRITE_DISPOSITION = WriteDisposition.WRITE_APPEND; + private static final Priority PRIORITY = Priority.BATCH; + private static final boolean ALLOW_LARGE_RESULTS = true; + private static final boolean USE_QUERY_CACHE = false; + private static final boolean FLATTEN_RESULTS = true; + private static final Integer MAX_BILLING_TIER = 123; + private static final Long MAX_BYTES_BILL = 12345L; + private static final List SCHEMA_UPDATE_OPTIONS = + ImmutableList.of(SchemaUpdateOption.ALLOW_FIELD_RELAXATION); + private static final List USER_DEFINED_FUNCTIONS = + ImmutableList.of(UserDefinedFunction.inline("Function"), UserDefinedFunction.fromUri("URI")); + private static final EncryptionConfiguration JOB_ENCRYPTION_CONFIGURATION = + EncryptionConfiguration.newBuilder().setKmsKeyName("KMS_KEY_1").build(); + private static final TimePartitioning TIME_PARTITIONING = + TimePartitioning.of(TimePartitioning.Type.DAY); + private static final Clustering CLUSTERING = + Clustering.newBuilder().setFields(ImmutableList.of("Foo", "Bar")).build(); + private static final Long TIMEOUT = 10L; + private static final RangePartitioning.Range RANGE = + RangePartitioning.Range.newBuilder().setStart(1L).setInterval(2L).setEnd(10L).build(); + private static final RangePartitioning RANGE_PARTITIONING = + RangePartitioning.newBuilder().setField("IntegerField").setRange(RANGE).build(); + + private static final ConnectionSettings CONNECTION_SETTINGS = + ConnectionSettings.newBuilder() + .setRequestTimeout(REQUEST_TIMEOUT) + .setNumBufferedRows(NUM_BUFFERED_ROWS) + .setMaxResults(MAX_RESULTS) + .setReadClientConnectionConfiguration(READ_CLIENT_CONNECTION_CONFIGURATION) + .setUseQueryCache(USE_QUERY_CACHE) + .setTableDefinitions(TABLE_DEFINITIONS) + .setAllowLargeResults(ALLOW_LARGE_RESULTS) + .setCreateDisposition(CREATE_DISPOSITION) + .setDefaultDataset(DATASET_ID) + .setDestinationTable(TABLE_ID) + .setWriteDisposition(WRITE_DISPOSITION) + .setPriority(PRIORITY) + .setFlattenResults(FLATTEN_RESULTS) + .setUserDefinedFunctions(USER_DEFINED_FUNCTIONS) + .setMaximumBillingTier(MAX_BILLING_TIER) + .setMaximumBytesBilled(MAX_BYTES_BILL) + .setSchemaUpdateOptions(SCHEMA_UPDATE_OPTIONS) + .setDestinationEncryptionConfiguration(JOB_ENCRYPTION_CONFIGURATION) + .setTimePartitioning(TIME_PARTITIONING) + .setClustering(CLUSTERING) + .setJobTimeoutMs(TIMEOUT) + .setRangePartitioning(RANGE_PARTITIONING) + .setConnectionProperties(CONNECTION_PROPERTIES) + .build(); + + @Test + public void testToBuilder() { + compareConnectionSettings(CONNECTION_SETTINGS, CONNECTION_SETTINGS.toBuilder().build()); + } + + @Test + public void testToBuilderIncomplete() { + ConnectionSettings connectionSettings = + ConnectionSettings.newBuilder().setDefaultDataset(DATASET_ID).build(); + compareConnectionSettings(connectionSettings, connectionSettings.toBuilder().build()); + } + + @Test + public void testBuilder() { + assertEquals(REQUEST_TIMEOUT, CONNECTION_SETTINGS.getRequestTimeout()); + assertEquals(NUM_BUFFERED_ROWS, CONNECTION_SETTINGS.getNumBufferedRows()); + assertEquals(MAX_RESULTS, CONNECTION_SETTINGS.getMaxResults()); + assertEquals( + READ_CLIENT_CONNECTION_CONFIGURATION, + CONNECTION_SETTINGS.getReadClientConnectionConfiguration()); + } + + private void compareConnectionSettings(ConnectionSettings expected, ConnectionSettings value) { + assertEquals(expected, value); + assertEquals(expected.hashCode(), value.hashCode()); + assertEquals(expected.toString(), value.toString()); + assertEquals(expected.getRequestTimeout(), value.getRequestTimeout()); + assertEquals(expected.getNumBufferedRows(), value.getNumBufferedRows()); + assertEquals(expected.getMaxResults(), value.getMaxResults()); + assertEquals( + expected.getReadClientConnectionConfiguration(), + value.getReadClientConnectionConfiguration()); + assertEquals(expected.getAllowLargeResults(), value.getAllowLargeResults()); + assertEquals(expected.getCreateDisposition(), value.getCreateDisposition()); + assertEquals(expected.getDefaultDataset(), value.getDefaultDataset()); + assertEquals(expected.getDestinationTable(), value.getDestinationTable()); + assertEquals(expected.getFlattenResults(), value.getFlattenResults()); + assertEquals(expected.getPriority(), value.getPriority()); + assertEquals(expected.getTableDefinitions(), value.getTableDefinitions()); + assertEquals(expected.getUseQueryCache(), value.getUseQueryCache()); + assertEquals(expected.getUserDefinedFunctions(), value.getUserDefinedFunctions()); + assertEquals(expected.getWriteDisposition(), value.getWriteDisposition()); + assertEquals(expected.getMaximumBillingTier(), value.getMaximumBillingTier()); + assertEquals(expected.getMaximumBytesBilled(), value.getMaximumBytesBilled()); + assertEquals(expected.getSchemaUpdateOptions(), value.getSchemaUpdateOptions()); + assertEquals( + expected.getDestinationEncryptionConfiguration(), + value.getDestinationEncryptionConfiguration()); + assertEquals(expected.getTimePartitioning(), value.getTimePartitioning()); + assertEquals(expected.getClustering(), value.getClustering()); + assertEquals(expected.getJobTimeoutMs(), value.getJobTimeoutMs()); + assertEquals(expected.getRangePartitioning(), value.getRangePartitioning()); + assertEquals(expected.getConnectionProperties(), value.getConnectionProperties()); + } +} From aaf9fa80859649e29aec6acf486b286fb1e3cd4b Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 28 Sep 2021 20:37:06 +0530 Subject: [PATCH 017/277] Fixed the producer logic for single page --- .../google/cloud/bigquery/ConnectionImpl.java | 54 ++++++++++++++++--- 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 91769b6e4..4bb88115d 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -22,8 +22,10 @@ import com.google.api.services.bigquery.model.*; import com.google.cloud.RetryHelper; import com.google.cloud.bigquery.spi.v2.BigQueryRpc; +import com.google.common.base.Function; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import java.util.*; import java.util.concurrent.BlockingQueue; @@ -90,6 +92,21 @@ public BigQueryResultSet executeSelect( return null; // TODO getQueryResultsRpc(JobId.fromPb(queryJob.getJobReference())); } + static class EndOfFieldValueList + extends AbstractList< + FieldValue> { // A reference of this class is used as a token to inform the thread + // consuming `buffer` BigQueryResultSetImpl that we have run out of records + @Override + public FieldValue get(int index) { + return null; + } + + @Override + public int size() { + return 0; + } + } + private BigQueryResultSet queryRpc(final String projectId, final QueryRequest queryRequest) { com.google.api.services.bigquery.model.QueryResponse results; try { @@ -126,6 +143,21 @@ private BigQueryResultSet queryRpc(final String projectId, final QueryRequest qu } } + private static Iterable getIterableFieldValueList( + Iterable tableDataPb, final Schema schema) { + return ImmutableList.copyOf( + Iterables.transform( + tableDataPb != null ? tableDataPb : ImmutableList.of(), + new Function() { + FieldList fields = schema != null ? schema.getFields() : null; + + @Override + public FieldValueList apply(TableRow rowPb) { + return FieldValueList.fromPb(rowPb.getF(), fields); + } + })); + } + private BigQueryResultSet processQueryResponseResults( com.google.api.services.bigquery.model.QueryResponse results) { Schema schema; @@ -133,17 +165,27 @@ private BigQueryResultSet processQueryResponseResults( schema = Schema.fromPb(results.getSchema()); numRows = results.getTotalRows().longValue(); + Iterable fieldValueLists = getIterableFieldValueList(results.getRows(), schema); + // Producer thread for populating the buffer row by row - BlockingQueue buffer = + BlockingQueue> buffer = new LinkedBlockingDeque<>(1000); // TODO: Update the capacity. Prefetch limit Runnable populateBufferRunnable = () -> { // producer thread populating the buffer - List tableRows = results.getRows(); - for (TableRow tableRow : tableRows) { // TODO: Handle the logic of pagination - buffer.offer(tableRow); + // List tableRows = results.getRows(); + for (FieldValueList fieldValueList : + fieldValueLists) { // TODO: Handle the logic of pagination + try { + buffer.put(fieldValueList); + } catch (InterruptedException e) { + e.printStackTrace(); + } } + if (results.getPageToken() == null) { - buffer.offer(null); // null marks the end of the records + buffer.offer( + new EndOfFieldValueList()); // A Hack to inform the BQResultSet that the blocking + // queue won't be populated with more records } }; Thread populateBufferWorker = new Thread(populateBufferRunnable); @@ -151,7 +193,7 @@ private BigQueryResultSet processQueryResponseResults( // only 1 page of result if (results.getPageToken() == null) { - return new BigQueryResultSetImpl(schema, numRows, buffer); + return new BigQueryResultSetImpl>(schema, numRows, buffer); } // use tabledata.list or Read API to fetch subsequent pages of results long totalRows = results.getTotalRows().longValue(); From c6871dc35fea7774edd48707ec1777fe9c4dd40f Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 28 Sep 2021 20:38:46 +0530 Subject: [PATCH 018/277] Fixed the producer logic for End of the stream case --- .../main/java/com/google/cloud/bigquery/ConnectionImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 4bb88115d..e5bd8eb8c 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -95,7 +95,7 @@ public BigQueryResultSet executeSelect( static class EndOfFieldValueList extends AbstractList< FieldValue> { // A reference of this class is used as a token to inform the thread - // consuming `buffer` BigQueryResultSetImpl that we have run out of records + // consuming `buffer` BigQueryResultSetImpl that we have run out of records @Override public FieldValue get(int index) { return null; @@ -185,7 +185,7 @@ private BigQueryResultSet processQueryResponseResults( if (results.getPageToken() == null) { buffer.offer( new EndOfFieldValueList()); // A Hack to inform the BQResultSet that the blocking - // queue won't be populated with more records + // queue won't be populated with more records } }; Thread populateBufferWorker = new Thread(populateBufferRunnable); From d049c859548fd9039687a415b2135c306c6344cb Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 28 Sep 2021 20:38:58 +0530 Subject: [PATCH 019/277] Fixed the consumer logic for next and getString methods --- .../cloud/bigquery/BigQueryResultSetImpl.java | 37 +++++++++++-------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index b91ed424c..7b98026ee 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -30,9 +30,8 @@ public class BigQueryResultSetImpl implements BigQueryResultSet { private final long totalRows; private final BlockingQueue buffer; private T cursor; - private ResultSetWrapper underlyingResultSet = null; + private final ResultSetWrapper underlyingResultSet; - // TODO : Implement a wrapper/struct like spanner public BigQueryResultSetImpl(Schema schema, long totalRows, BlockingQueue buffer) { this.schema = schema; this.totalRows = totalRows; @@ -59,24 +58,31 @@ private class ResultSetWrapper extends AbstractJdbcResultSet { @Override /*Advances the result set to the next row, returning false if no such row exists. Potentially blocking operation*/ public boolean next() throws SQLException { - if (buffer.peek() == null) { // producer has no more rows left. - return false; - } try { cursor = buffer.take(); // advance the cursor,Potentially blocking operation + if (isEndOfStream(cursor)) { // check for end of stream + cursor = null; + return false; + } } catch (InterruptedException e) { - throw new SQLException("No rows left"); + throw new SQLException("Error occurred while reading buffer"); } return true; } + private boolean isEndOfStream(T cursor) { + return cursor + instanceof + ConnectionImpl.EndOfFieldValueList; // TODO: Similar check will be required for Arrow + } + @Override public Object getObject(String fieldName) throws SQLException { - if (cursor instanceof TableRow) { + if (cursor != null && cursor instanceof TableRow) { TableRow currentRow = (TableRow) cursor; - if (currentRow == null) { - throw new SQLException("No rows left"); + if (fieldName == null) { + throw new SQLException("fieldName can't be null"); } return currentRow.get(fieldName); } @@ -86,14 +92,15 @@ public Object getObject(String fieldName) throws SQLException { @Override public String getString(String fieldName) throws SQLException { - Object value = getObject(fieldName); - if (value == null) { + if (fieldName == null) { throw new SQLException("fieldName can't be null"); - } else if (!(value instanceof String)) { - throw new SQLException("value not in instance of String"); - } else { - return (String) value; } + if (cursor == null) { + return null; + } else if (cursor instanceof FieldValueList) { + return ((FieldValueList) cursor).get(fieldName).getStringValue(); + } + return null; // TODO: Implementation for Arrow } @Override From c2160aea585e06bce07e305b969e12531fb565e9 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 29 Sep 2021 12:15:31 +0530 Subject: [PATCH 020/277] Implemented getInt --- .../cloud/bigquery/BigQueryResultSetImpl.java | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index 7b98026ee..d97185513 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -103,6 +103,20 @@ public String getString(String fieldName) throws SQLException { return null; // TODO: Implementation for Arrow } + + @Override + public int getInt(String fieldName) throws SQLException { + if (fieldName == null) { + throw new SQLException("fieldName can't be null"); + } + if (cursor == null) { + return 0;//the column value; if the value is SQL NULL, the value returned is 0 as per java.sql.ResultSet definition + } else if (cursor instanceof FieldValueList) { + return ((FieldValueList) cursor).get(fieldName).getNumericValue().intValue(); + } + return 0; // TODO: Implementation for Arrow + } + @Override public long getLong(String fieldName) throws SQLException { Object value = getObject(fieldName); @@ -180,16 +194,5 @@ public Timestamp getTimestamp(String fieldName) throws SQLException { } } - @Override - public int getInt(String fieldName) throws SQLException { - Object value = getObject(fieldName); - if (value == null) { - throw new SQLException("fieldName can't be null"); - } else if (!(value instanceof Integer || value instanceof String)) { - throw new SQLException("value cannot be converted to int"); - } else { - return Integer.parseInt(String.valueOf(value)); - } - } } } From c7da28fbb0b34976c35b44facc85fee626839ff1 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 29 Sep 2021 12:43:34 +0530 Subject: [PATCH 021/277] Fixed the consumer logic for next and getString methods - lint --- .../com/google/cloud/bigquery/BigQueryResultSetImpl.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index d97185513..575402364 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -103,14 +103,14 @@ public String getString(String fieldName) throws SQLException { return null; // TODO: Implementation for Arrow } - @Override public int getInt(String fieldName) throws SQLException { if (fieldName == null) { throw new SQLException("fieldName can't be null"); } if (cursor == null) { - return 0;//the column value; if the value is SQL NULL, the value returned is 0 as per java.sql.ResultSet definition + return 0; // the column value; if the value is SQL NULL, the value returned is 0 as per + // java.sql.ResultSet definition } else if (cursor instanceof FieldValueList) { return ((FieldValueList) cursor).get(fieldName).getNumericValue().intValue(); } @@ -193,6 +193,5 @@ public Timestamp getTimestamp(String fieldName) throws SQLException { return Timestamp.valueOf(String.valueOf(value)); } } - } } From ac16aec9a4f588222ed4b7a0f9e258b511c86813 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 29 Sep 2021 12:43:53 +0530 Subject: [PATCH 022/277] fixed testExecuteQuerySinglePageTableRow --- .../google/cloud/bigquery/it/ITBigQueryTest.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 5a24682b4..41671246c 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -1958,17 +1958,21 @@ public void testQueryExternalHivePartitioningOptionCustomLayout() throws Interru @Test public void testExecuteQuerySinglePageTableRow() throws SQLException { - String query = "SELECT * FROM " + TABLE_ID_FASTQUERY.getTable(); + String query = + "SELECT TimestampField, StringField, BooleanField FROM " + TABLE_ID_FASTQUERY.getTable(); ConnectionSettings connectionSettings = ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); Connection connection = bigquery.createConnection(connectionSettings); BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); ResultSet rs = bigQueryResultSet.getResultSet(); - while (rs.next()) { - // System.out.println(rs.getString("TimestampField")); - } - // assertEquals(QUERY_RESULT_SCHEMA, bigQueryResultSet.getSchema()); - // assertEquals(2, bigQueryResultSet.getTotalRows()); + // while (rs.next()) { + // System.out.println(rs.getString("TimestampField")); + // } + assertEquals(QUERY_RESULT_SCHEMA, bigQueryResultSet.getSchema()); + assertEquals(2, bigQueryResultSet.getTotalRows()); + assertTrue(rs.next()); + assertTrue(rs.next()); + assertFalse(rs.next()); // it should return false for the third call as the table has 2 rows } @Test From 8b14422d0b7b38604844bcc1f5a67461ca28140b Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Mon, 27 Sep 2021 21:22:56 +0200 Subject: [PATCH 023/277] deps: update dependency com.google.cloud:google-cloud-storage to v2.1.6 (#1621) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [![WhiteSource Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com) This PR contains the following updates: | Package | Change | Age | Adoption | Passing | Confidence | |---|---|---|---|---|---| | [com.google.cloud:google-cloud-storage](https://togithub.com/googleapis/java-storage) | `2.1.5` -> `2.1.6` | [![age](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-storage/2.1.6/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-storage/2.1.6/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-storage/2.1.6/compatibility-slim/2.1.5)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-storage/2.1.6/confidence-slim/2.1.5)](https://docs.renovatebot.com/merge-confidence/) | --- ### Release Notes
googleapis/java-storage ### [`v2.1.6`](https://togithub.com/googleapis/java-storage/blob/master/CHANGELOG.md#​216-httpswwwgithubcomgoogleapisjava-storagecomparev215v216-2021-09-24) [Compare Source](https://togithub.com/googleapis/java-storage/compare/v2.1.5...v2.1.6)
--- ### Configuration 📅 **Schedule**: At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box. --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#github/googleapis/java-bigquery). --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4398e2a92..c1b038073 100644 --- a/pom.xml +++ b/pom.xml @@ -114,7 +114,7 @@ com.google.cloud google-cloud-storage - 2.1.5 + 2.1.6 test From 0eac1a8178c735167bae34d7860a2039a00c9dd7 Mon Sep 17 00:00:00 2001 From: Emily Ball Date: Tue, 28 Sep 2021 12:42:16 -0700 Subject: [PATCH 024/277] chore: change branch master to main (#1623) --- .github/sync-repo-settings.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index ab4807cbe..b595a8a2c 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -2,7 +2,7 @@ rebaseMergeAllowed: true squashMergeAllowed: true mergeCommitAllowed: false branchProtectionRules: - - pattern: master + - pattern: main isAdminEnforced: true requiredApprovingReviewCount: 1 requiresCodeOwnerReviews: true From 4bc9d908933c075177a25dea9cb2dabc85099c4d Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 30 Sep 2021 08:13:42 +0530 Subject: [PATCH 025/277] Updated gitignore --- .gitignore | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 8782d86f6..e9a78d1ef 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,8 @@ __pycache__ .settings .classpath .DS_Store - +.diff.txt +.new-list.txt +.org-list.txt +.flattened-pom.xml +SimpleBenchmarkApp/src/main/java/com/google/cloud/App.java \ No newline at end of file From 400479d92da17d5390eef0a36cfba1a3b41e1ab8 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 30 Sep 2021 09:08:22 +0530 Subject: [PATCH 026/277] Deleting --- .../src/main/java/com/google/cloud/App.java | 114 --- google-cloud-bigquery/.diff.txt | 0 google-cloud-bigquery/.flattened-pom.xml | 789 ------------------ google-cloud-bigquery/.new-list.txt | 76 -- google-cloud-bigquery/.org-list.txt | 76 -- 5 files changed, 1055 deletions(-) delete mode 100644 SimpleBenchmarkApp/src/main/java/com/google/cloud/App.java delete mode 100644 google-cloud-bigquery/.diff.txt delete mode 100644 google-cloud-bigquery/.flattened-pom.xml delete mode 100644 google-cloud-bigquery/.new-list.txt delete mode 100644 google-cloud-bigquery/.org-list.txt diff --git a/SimpleBenchmarkApp/src/main/java/com/google/cloud/App.java b/SimpleBenchmarkApp/src/main/java/com/google/cloud/App.java deleted file mode 100644 index e4d2d9394..000000000 --- a/SimpleBenchmarkApp/src/main/java/com/google/cloud/App.java +++ /dev/null @@ -1,114 +0,0 @@ -package com.google.cloud; - -import com.google.auth.oauth2.GoogleCredentials; -import com.google.auth.oauth2.ServiceAccountCredentials; -import com.google.cloud.bigquery.BigQuery; -import com.google.cloud.bigquery.BigQueryOptions; -import com.google.cloud.bigquery.QueryJobConfiguration; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.UUID; - -// Launcher for single query execution -public class App { - - private static final String project = "bigquery-smallqueries-ram"; - private static final String dataset = "turbosql"; - - private static final String query = "select\n" - + "sum(NASDelay) as C1,\n" - + "count(NASDelay) as C2\n" - + "from\n" - + "(\n" - + " select\n" - + " NASDelay\n" - + " from\n" - + " flight_data\n" - + " where\n" - + " (\n" - + " cast(FlightDate as TIMESTAMP) < TIMESTAMP('2018-12-31 12:00:01')\n" - + " and\n" - + " cast(FlightDate as TIMESTAMP) >= TIMESTAMP('2018-01-01 00:00:00')\n" - + " )\n" - + " and\n" - + " Origin = 'LAS'\n" - + ")\n" - + "as ITBL"; - - private static final String credentials = "{\n" - + " \"type\": \"service_account\",\n" - + " \"project_id\": \"bigquery-smallqueries-ram\",\n" - + " \"private_key_id\": \"1e08ae70aedd777112ee5b0c4246e14a2c1778d2\",\n" - + " \"private_key\": \"-----BEGIN PRIVATE KEY-----\\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCre4kgo2QFnblN\\n3qan0j+tISBK+Y/lgOCuMcSP2JVET+vu4GzYGjXOk+ciEETDVm2RO6RT91yxkGou\\n41cEyafsVavYwSOaE13sTJ+rBq62WI3tJjIXY02ujq2rIacDkSvNFSArxFh4g5y/\\npOlP1PDBEfJ+BW/n7il6STyfaHY9DeiDqRuD/Ru/FptjpsHtxZF0f3hVBN50tqNE\\nBsDOERN3YJUd/DXE9LYtsOjdnehZO6u7aGB7y9oZOZPrQHV+cOidd8vRMja4qtnq\\nHphB0DBWAWhVsbpz/3YXRIazbpN172qDR4IeGWcAk+dB1wW+fozXeBMGdzCXmQ3s\\nW9kgjHLRAgMBAAECggEAEGfrQKIhKaBjYB3TF+83hFrM+h9SMwTXehLs2U5BSZ3T\\n1rDF+Q7Y4wCndkzUJ1QXe3OXTyk1Rw+n+QBrDNw2Ipg3rq8bttvvenICPJyRDoT+\\nXxfuZuprPi4MU3kOv3qeFCrXPxiz1iPRVUxzvd2DgxUye/gNMsVIVpi3IhEebszD\\ntIk2DNVR8blP3iHvG627oA6n1xlyRDgtuYeTcOoLE8QGL6BuWrG7CCbXuVbB3ARc\\nzUoWWkNOgbZL74LRUdPHraPtRv1zhOlqMotCz+K0mu6EzfIWAzZ8K3HiWH2Qc40j\\ncfLdXOwN/xJRlAVAWGmWW6dbi/qrj9kh+zCAEv0vOQKBgQDb+kYyybjE2YglbT42\\nipsLz0IfijhD4xSD54SIDST48QHHJtMg0/nH3tQmFn81ZqF2SUeTp7aVlwyQl/wS\\nkEQUa3IiQ5LPanuH7FtbhnWC5xzcGDcSnnu7RMtMF+Us66PPD0cSIjtZQ7MbCj2D\\nigcQqnGbcfnNrTN6F4+Eqi1p9wKBgQDHkEo5Z5KJZpuK+LPySwrd7VUyA6muBG1f\\njpgRDCanOwk6bCb1nr8WFKUTH1+RmWwJxcq/GHIAHlsqHvmNnI2dWCw/2XQvb6B0\\nNTsjc9dEwvSQTdstibcJYXSqI1a70/fdzBlKL0EmWk2fSoK1/YNgMqvbzWWkd2Sa\\nwlLaE60XdwKBgFSqVyplCYB6WTROf4tufY5mDwUkpdM7K0I5cYELzhcia5TDFK+l\\n5pVO5khikEN1ZN+qBKqH+nZI1MUyOgrLC+jwEdWuPGsoiLMf/WmUrtXbLfhoOYop\\nBWZma/i1mbdYWovvTWNlWYJZ1C2sG1DtZxq6/07c51CKQS3Us6BT/3axAoGAbo6G\\nQEUbzlj05MmhTyK5s3bvEtUqpIg5W43wuskDhPPUyfPupXY7oGzxgqWH2W6ohsV3\\n6+QMC/rFQJGGaSiI39lgMkMy9bCesKJoz9w2LxbeTC+FtDWuHFlMO5F2VHo6wDp4\\n7Ds/mZK/m/a4cUAwDxQjV5Lzs2idaIstQTlTVEMCgYBfqABqnx1Nm9FAgheD1rpL\\nZQlYiOyKPdOfyE1KF2uzKVE4FAmGmKtCvfFVc979CNQEPlmve5GiS8GRIG0FKlU+\\nHYO5KN5XezuH+osoMgbCnzXrbAgwHcZ/pYLqPPi85xKYTY+c+wUmJBl5q3Qpp8VR\\nAxWLJJGjiztrcpiTbD9DIg==\\n-----END PRIVATE KEY-----\\n\",\n" - + " \"client_email\": \"temp-testing@bigquery-smallqueries-ram.iam.gserviceaccount.com\",\n" - + " \"client_id\": \"107395172672076342797\",\n" - + " \"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\n" - + " \"token_uri\": \"https://oauth2.googleapis.com/token\",\n" - + " \"auth_provider_x509_cert_url\": \"https://www.googleapis.com/oauth2/v1/certs\",\n" - + " \"client_x509_cert_url\": \"https://www.googleapis.com/robot/v1/metadata/x509/temp-testing%40bigquery-smallqueries-ram.iam.gserviceaccount.com\"\n" - + "}\n"; - - private BigQuery bigQuery; - - public static void main( String[] args ) { - App app = new App(); - app.benchmarkQuery(); - } - - /** - * Method to create connection and benchmark a single query on BigQuery. - */ - private void benchmarkQuery() { - try { - ServiceAccountCredentials serviceAccountCredentials = - ServiceAccountCredentials.fromStream( - new ByteArrayInputStream(credentials.getBytes()) - ); - GoogleCredentials googleCredentials = serviceAccountCredentials.createScoped("https://www.googleapis.com/auth/cloud-platform"); - googleCredentials.refreshIfExpired(); - this.bigQuery = BigQueryOptions.newBuilder() - .setCredentials(googleCredentials) - .setProjectId(project) - .build().getService(); - } catch (IOException e) { - System.out.println("Failed to prepare execution." + e.getMessage()); - } - - try { - this.executeQuery(); - } catch (InterruptedException e) { - System.out.println("Execution failure."); - } - } - - /** - * Benchmark a query on BigQuery and return performance details. - * - * @return A serialized json string containing the query performance details. - * @throws InterruptedException - */ - public void executeQuery() throws InterruptedException { - String queryId = UUID.randomUUID().toString(); - List results = new ArrayList<>(); - for(int i=0; i<=2; i++) { - // Use standard SQL syntax for queries. - // See: https://cloud.google.com/bigquery/sql-reference/ - QueryJobConfiguration queryConfig = - QueryJobConfiguration.newBuilder(this.query) - .setUseLegacySql(false) - .setUseQueryCache(false) - .setDefaultDataset(this.dataset) - .setLabels(Collections.singletonMap("query_id", queryId)) - .build(); - long start = System.currentTimeMillis(); - this.bigQuery.query(queryConfig); - long end = System.currentTimeMillis(); - double executionTimeInSecs = (end - start) / 1000.0; - results.add(executionTimeInSecs); - } - System.out.println("query_wall_time_in_secs: " + results); - } -} diff --git a/google-cloud-bigquery/.diff.txt b/google-cloud-bigquery/.diff.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/google-cloud-bigquery/.flattened-pom.xml b/google-cloud-bigquery/.flattened-pom.xml deleted file mode 100644 index 8f23d89b3..000000000 --- a/google-cloud-bigquery/.flattened-pom.xml +++ /dev/null @@ -1,789 +0,0 @@ - - - 4.0.0 - com.google.cloud - google-cloud-bigquery - 2.1.8-SNAPSHOT - BigQuery - BigQuery - https://github.com/googleapis/java-bigquery - - Google LLC - - - - Apache-2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt - - - - - chingor - Jeff Ching - chingor@google.com - Google - - Developer - - - - - scm:git:git@github.com:googleapis/java-bigquery.git/google-cloud-bigquery - scm:git:git@github.com:googleapis/java-bigquery.git/google-cloud-bigquery - https://github.com/googleapis/java-bigquery/google-cloud-bigquery - - - GitHub Issues - https://github.com/googleapis/java-bigquery/issues - - - - sonatype-nexus-staging - https://google.oss.sonatype.org/service/local/staging/deploy/maven2/ - - - sonatype-nexus-snapshots - https://google.oss.sonatype.org/content/repositories/snapshots - - - - - com.google.cloud - google-cloud-core - 2.1.2 - compile - - - guava - com.google.guava - - - gax - com.google.api - - - auto-value-annotations - com.google.auto.value - - - protobuf-java-util - com.google.protobuf - - - proto-google-common-protos - com.google.api.grpc - - - proto-google-iam-v1 - com.google.api.grpc - - - threetenbp - org.threeten - - - api-common - com.google.api - - - google-auth-library-credentials - com.google.auth - - - google-auth-library-oauth2-http - com.google.auth - - - google-http-client - com.google.http-client - - - google-http-client-gson - com.google.http-client - - - protobuf-java - com.google.protobuf - - - jsr305 - com.google.code.findbugs - - - false - - - com.google.protobuf - protobuf-java-util - 3.17.3 - compile - - - protobuf-java - com.google.protobuf - - - guava - com.google.guava - - - error_prone_annotations - com.google.errorprone - - - gson - com.google.code.gson - - - false - - - com.google.code.gson - gson - 2.8.8 - compile - false - - - com.google.api.grpc - proto-google-common-protos - 2.5.0 - compile - - - protobuf-java - com.google.protobuf - - - false - - - com.google.api.grpc - proto-google-iam-v1 - 1.1.0 - compile - - - protobuf-java - com.google.protobuf - - - proto-google-common-protos - com.google.api.grpc - - - false - - - com.google.auth - google-auth-library-credentials - 1.1.0 - compile - false - - - com.google.http-client - google-http-client-gson - 1.40.0 - compile - - - google-http-client - com.google.http-client - - - gson - com.google.code.gson - - - false - - - com.google.protobuf - protobuf-java - 3.17.3 - compile - false - - - com.google.cloud - google-cloud-core-http - 2.1.2 - compile - - - google-cloud-core - com.google.cloud - - - google-auth-library-credentials - com.google.auth - - - google-auth-library-oauth2-http - com.google.auth - - - google-http-client - com.google.http-client - - - guava - com.google.guava - - - google-api-client - com.google.api-client - - - google-http-client-appengine - com.google.http-client - - - gax - com.google.api - - - gax-httpjson - com.google.api - - - jsr305 - com.google.code.findbugs - - - opencensus-api - io.opencensus - - - opencensus-contrib-http-util - io.opencensus - - - api-common - com.google.api - - - false - - - com.google.api-client - google-api-client - 1.32.1 - compile - - - google-oauth-client - com.google.oauth-client - - - google-http-client-gson - com.google.http-client - - - guava - com.google.guava - - - google-http-client-apache-v2 - com.google.http-client - - - httpcore - org.apache.httpcomponents - - - httpclient - org.apache.httpcomponents - - - google-http-client - com.google.http-client - - - false - - - com.google.oauth-client - google-oauth-client - 1.32.1 - compile - - - google-http-client - com.google.http-client - - - guava - com.google.guava - - - false - - - com.google.http-client - google-http-client-apache-v2 - 1.40.0 - compile - - - google-http-client - com.google.http-client - - - httpclient - org.apache.httpcomponents - - - httpcore - org.apache.httpcomponents - - - false - - - com.google.http-client - google-http-client-appengine - 1.40.0 - compile - - - google-http-client - com.google.http-client - - - appengine-api-1.0-sdk - com.google.appengine - - - false - - - com.google.api - gax-httpjson - 0.89.0 - compile - - - gax - com.google.api - - - protobuf-java - com.google.protobuf - - - protobuf-java-util - com.google.protobuf - - - gson - com.google.code.gson - - - guava - com.google.guava - - - jsr305 - com.google.code.findbugs - - - threetenbp - org.threeten - - - google-http-client - com.google.http-client - - - google-http-client-gson - com.google.http-client - - - google-auth-library-oauth2-http - com.google.auth - - - google-auth-library-credentials - com.google.auth - - - proto-google-common-protos - com.google.api.grpc - - - false - - - io.opencensus - opencensus-api - 0.28.0 - compile - - - grpc-context - io.grpc - - - false - - - io.grpc - grpc-context - 1.40.1 - compile - false - - - io.opencensus - opencensus-contrib-http-util - 0.28.0 - compile - - - opencensus-api - io.opencensus - - - guava - com.google.guava - - - false - - - com.google.http-client - google-http-client-jackson2 - 1.40.0 - compile - - - google-http-client - com.google.http-client - - - jackson-core - com.fasterxml.jackson.core - - - false - - - com.fasterxml.jackson.core - jackson-core - 2.12.5 - compile - false - - - com.google.auto.value - auto-value-annotations - 1.8.2 - provided - false - - - com.google.http-client - google-http-client - 1.40.0 - compile - - - httpclient - org.apache.httpcomponents - - - httpcore - org.apache.httpcomponents - - - jsr305 - com.google.code.findbugs - - - guava - com.google.guava - - - j2objc-annotations - com.google.j2objc - - - opencensus-api - io.opencensus - - - opencensus-contrib-http-util - io.opencensus - - - false - - - org.apache.httpcomponents - httpclient - 4.5.13 - compile - - - httpcore - org.apache.httpcomponents - - - commons-logging - commons-logging - - - commons-codec - commons-codec - - - false - - - commons-logging - commons-logging - 1.2 - compile - - - log4j - log4j - - - logkit - logkit - - - avalon-framework - avalon-framework - - - servlet-api - javax.servlet - - - false - - - commons-codec - commons-codec - 1.15 - compile - false - - - org.apache.httpcomponents - httpcore - 4.4.14 - compile - false - - - com.google.j2objc - j2objc-annotations - 1.3 - compile - false - - - org.checkerframework - checker-compat-qual - 2.5.5 - compile - false - - - com.google.auth - google-auth-library-oauth2-http - 1.1.0 - compile - - - auto-value-annotations - com.google.auto.value - - - jsr305 - com.google.code.findbugs - - - google-auth-library-credentials - com.google.auth - - - google-http-client - com.google.http-client - - - google-http-client-gson - com.google.http-client - - - guava - com.google.guava - - - false - - - com.google.code.findbugs - jsr305 - 3.0.2 - compile - false - - - com.google.apis - google-api-services-bigquery - v2-rev20210828-1.32.1 - compile - - - google-api-client - com.google.api-client - - - false - - - com.google.api - api-common - 2.0.2 - compile - - - guava - com.google.guava - - - jsr305 - com.google.code.findbugs - - - javax.annotation-api - javax.annotation - - - auto-value-annotations - com.google.auto.value - - - false - - - javax.annotation - javax.annotation-api - 1.3.2 - compile - false - - - com.google.guava - guava - 30.1.1-jre - compile - - - failureaccess - com.google.guava - - - listenablefuture - com.google.guava - - - jsr305 - com.google.code.findbugs - - - checker-qual - org.checkerframework - - - error_prone_annotations - com.google.errorprone - - - j2objc-annotations - com.google.j2objc - - - srczip - jdk - - - false - - - com.google.guava - failureaccess - 1.0.1 - compile - false - - - com.google.guava - listenablefuture - 9999.0-empty-to-avoid-conflict-with-guava - compile - false - - - org.checkerframework - checker-qual - 3.8.0 - compile - false - - - com.google.api - gax - 2.4.0 - compile - - - guava - com.google.guava - - - jsr305 - com.google.code.findbugs - - - threetenbp - org.threeten - - - google-auth-library-oauth2-http - com.google.auth - - - api-common - com.google.api - - - opencensus-api - io.opencensus - - - false - - - org.threeten - threetenbp - 1.5.1 - compile - false - - - com.google.errorprone - error_prone_annotations - 2.9.0 - compile - false - - - diff --git a/google-cloud-bigquery/.new-list.txt b/google-cloud-bigquery/.new-list.txt deleted file mode 100644 index 79da80abc..000000000 --- a/google-cloud-bigquery/.new-list.txt +++ /dev/null @@ -1,76 +0,0 @@ -[INFO] com.fasterxml.jackson.core:jackson-annotations:jar:2.12.4:compile -[INFO] com.fasterxml.jackson.core:jackson-core:jar:2.12.4:compile -[INFO] com.fasterxml.jackson.core:jackson-databind:jar:2.12.4:compile -[INFO] com.google.android:annotations:jar:4.1.1.4:runtime -[INFO] com.google.api-client:google-api-client:jar:1.32.1:compile -[INFO] com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:jar:2.0.1:compile -[INFO] com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:jar:0.124.1:compile -[INFO] com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:jar:0.124.1:compile -[INFO] com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:jar:2.0.1:compile -[INFO] com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:jar:0.124.1:compile -[INFO] com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:jar:0.124.1:compile -[INFO] com.google.api.grpc:proto-google-common-protos:jar:2.3.2:compile -[INFO] com.google.api.grpc:proto-google-iam-v1:jar:1.0.14:compile -[INFO] com.google.api:api-common:jar:2.0.1:compile -[INFO] com.google.api:gax-grpc:jar:2.1.0:compile -[INFO] com.google.api:gax-httpjson:jar:0.86.0:compile -[INFO] com.google.api:gax:jar:2.1.0:compile -[INFO] com.google.apis:google-api-services-bigquery:jar:v2-rev20210726-1.32.1:compile -[INFO] com.google.auth:google-auth-library-credentials:jar:1.0.0:compile -[INFO] com.google.auth:google-auth-library-oauth2-http:jar:1.0.0:compile -[INFO] com.google.cloud:google-cloud-bigquerystorage:jar:2.0.1:compile -[INFO] com.google.cloud:google-cloud-core-http:jar:2.0.5:compile -[INFO] com.google.cloud:google-cloud-core:jar:2.0.5:compile -[INFO] com.google.code.findbugs:jsr305:jar:3.0.2:compile -[INFO] com.google.code.gson:gson:jar:2.8.7:compile -[INFO] com.google.errorprone:error_prone_annotations:jar:2.8.1:compile -[INFO] com.google.flatbuffers:flatbuffers-java:jar:1.12.0:compile -[INFO] com.google.guava:failureaccess:jar:1.0.1:compile -[INFO] com.google.guava:guava:jar:30.1.1-jre:compile -[INFO] com.google.guava:listenablefuture:jar:9999.0-empty-to-avoid-conflict-with-guava:compile -[INFO] com.google.http-client:google-http-client-apache-v2:jar:1.39.2:compile -[INFO] com.google.http-client:google-http-client-appengine:jar:1.39.2:compile -[INFO] com.google.http-client:google-http-client-gson:jar:1.39.2:compile -[INFO] com.google.http-client:google-http-client-jackson2:jar:1.39.2:compile -[INFO] com.google.http-client:google-http-client:jar:1.39.2:compile -[INFO] com.google.j2objc:j2objc-annotations:jar:1.3:compile -[INFO] com.google.oauth-client:google-oauth-client:jar:1.31.5:compile -[INFO] com.google.protobuf:protobuf-java-util:jar:3.17.3:compile -[INFO] com.google.protobuf:protobuf-java:jar:3.17.3:compile -[INFO] commons-codec:commons-codec:jar:1.15:compile -[INFO] commons-logging:commons-logging:jar:1.2:compile -[INFO] io.grpc:grpc-alts:jar:1.39.0:compile -[INFO] io.grpc:grpc-api:jar:1.39.0:compile -[INFO] io.grpc:grpc-auth:jar:1.39.0:compile -[INFO] io.grpc:grpc-context:jar:1.39.0:compile -[INFO] io.grpc:grpc-core:jar:1.39.0:compile -[INFO] io.grpc:grpc-grpclb:jar:1.39.0:compile -[INFO] io.grpc:grpc-netty-shaded:jar:1.39.0:compile -[INFO] io.grpc:grpc-protobuf-lite:jar:1.39.0:compile -[INFO] io.grpc:grpc-protobuf:jar:1.39.0:compile -[INFO] io.grpc:grpc-stub:jar:1.39.0:compile -[INFO] io.netty:netty-buffer:jar:4.1.48.Final:runtime -[INFO] io.netty:netty-common:jar:4.1.48.Final:compile -[INFO] io.opencensus:opencensus-api:jar:0.28.0:compile -[INFO] io.opencensus:opencensus-contrib-http-util:jar:0.28.0:compile -[INFO] io.perfmark:perfmark-api:jar:0.23.0:runtime -[INFO] javax.annotation:javax.annotation-api:jar:1.3.2:compile -[INFO] net.sf.jopt-simple:jopt-simple:jar:4.6:compile -[INFO] org.apache.arrow:arrow-format:jar:5.0.0:compile -[INFO] org.apache.arrow:arrow-memory-core:jar:5.0.0:compile -[INFO] org.apache.arrow:arrow-memory-netty:jar:5.0.0:runtime -[INFO] org.apache.arrow:arrow-vector:jar:5.0.0:compile -[INFO] org.apache.avro:avro:jar:1.10.2:compile -[INFO] org.apache.commons:commons-compress:jar:1.20:compile -[INFO] org.apache.commons:commons-math3:jar:3.2:compile -[INFO] org.apache.httpcomponents:httpclient:jar:4.5.13:compile -[INFO] org.apache.httpcomponents:httpcore:jar:4.4.14:compile -[INFO] org.checkerframework:checker-compat-qual:jar:2.5.5:compile -[INFO] org.checkerframework:checker-qual:jar:3.8.0:compile -[INFO] org.codehaus.mojo:animal-sniffer-annotations:jar:1.20:runtime -[INFO] org.conscrypt:conscrypt-openjdk-uber:jar:2.5.1:compile -[INFO] org.json:json:jar:20200518:compile -[INFO] org.openjdk.jmh:jmh-core:jar:1.32:compile -[INFO] org.openjdk.jmh:jmh-generator-annprocess:jar:1.32:compile -[INFO] org.slf4j:slf4j-api:jar:1.7.30:compile -[INFO] org.threeten:threetenbp:jar:1.5.1:compile diff --git a/google-cloud-bigquery/.org-list.txt b/google-cloud-bigquery/.org-list.txt deleted file mode 100644 index 79da80abc..000000000 --- a/google-cloud-bigquery/.org-list.txt +++ /dev/null @@ -1,76 +0,0 @@ -[INFO] com.fasterxml.jackson.core:jackson-annotations:jar:2.12.4:compile -[INFO] com.fasterxml.jackson.core:jackson-core:jar:2.12.4:compile -[INFO] com.fasterxml.jackson.core:jackson-databind:jar:2.12.4:compile -[INFO] com.google.android:annotations:jar:4.1.1.4:runtime -[INFO] com.google.api-client:google-api-client:jar:1.32.1:compile -[INFO] com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:jar:2.0.1:compile -[INFO] com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:jar:0.124.1:compile -[INFO] com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:jar:0.124.1:compile -[INFO] com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:jar:2.0.1:compile -[INFO] com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:jar:0.124.1:compile -[INFO] com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:jar:0.124.1:compile -[INFO] com.google.api.grpc:proto-google-common-protos:jar:2.3.2:compile -[INFO] com.google.api.grpc:proto-google-iam-v1:jar:1.0.14:compile -[INFO] com.google.api:api-common:jar:2.0.1:compile -[INFO] com.google.api:gax-grpc:jar:2.1.0:compile -[INFO] com.google.api:gax-httpjson:jar:0.86.0:compile -[INFO] com.google.api:gax:jar:2.1.0:compile -[INFO] com.google.apis:google-api-services-bigquery:jar:v2-rev20210726-1.32.1:compile -[INFO] com.google.auth:google-auth-library-credentials:jar:1.0.0:compile -[INFO] com.google.auth:google-auth-library-oauth2-http:jar:1.0.0:compile -[INFO] com.google.cloud:google-cloud-bigquerystorage:jar:2.0.1:compile -[INFO] com.google.cloud:google-cloud-core-http:jar:2.0.5:compile -[INFO] com.google.cloud:google-cloud-core:jar:2.0.5:compile -[INFO] com.google.code.findbugs:jsr305:jar:3.0.2:compile -[INFO] com.google.code.gson:gson:jar:2.8.7:compile -[INFO] com.google.errorprone:error_prone_annotations:jar:2.8.1:compile -[INFO] com.google.flatbuffers:flatbuffers-java:jar:1.12.0:compile -[INFO] com.google.guava:failureaccess:jar:1.0.1:compile -[INFO] com.google.guava:guava:jar:30.1.1-jre:compile -[INFO] com.google.guava:listenablefuture:jar:9999.0-empty-to-avoid-conflict-with-guava:compile -[INFO] com.google.http-client:google-http-client-apache-v2:jar:1.39.2:compile -[INFO] com.google.http-client:google-http-client-appengine:jar:1.39.2:compile -[INFO] com.google.http-client:google-http-client-gson:jar:1.39.2:compile -[INFO] com.google.http-client:google-http-client-jackson2:jar:1.39.2:compile -[INFO] com.google.http-client:google-http-client:jar:1.39.2:compile -[INFO] com.google.j2objc:j2objc-annotations:jar:1.3:compile -[INFO] com.google.oauth-client:google-oauth-client:jar:1.31.5:compile -[INFO] com.google.protobuf:protobuf-java-util:jar:3.17.3:compile -[INFO] com.google.protobuf:protobuf-java:jar:3.17.3:compile -[INFO] commons-codec:commons-codec:jar:1.15:compile -[INFO] commons-logging:commons-logging:jar:1.2:compile -[INFO] io.grpc:grpc-alts:jar:1.39.0:compile -[INFO] io.grpc:grpc-api:jar:1.39.0:compile -[INFO] io.grpc:grpc-auth:jar:1.39.0:compile -[INFO] io.grpc:grpc-context:jar:1.39.0:compile -[INFO] io.grpc:grpc-core:jar:1.39.0:compile -[INFO] io.grpc:grpc-grpclb:jar:1.39.0:compile -[INFO] io.grpc:grpc-netty-shaded:jar:1.39.0:compile -[INFO] io.grpc:grpc-protobuf-lite:jar:1.39.0:compile -[INFO] io.grpc:grpc-protobuf:jar:1.39.0:compile -[INFO] io.grpc:grpc-stub:jar:1.39.0:compile -[INFO] io.netty:netty-buffer:jar:4.1.48.Final:runtime -[INFO] io.netty:netty-common:jar:4.1.48.Final:compile -[INFO] io.opencensus:opencensus-api:jar:0.28.0:compile -[INFO] io.opencensus:opencensus-contrib-http-util:jar:0.28.0:compile -[INFO] io.perfmark:perfmark-api:jar:0.23.0:runtime -[INFO] javax.annotation:javax.annotation-api:jar:1.3.2:compile -[INFO] net.sf.jopt-simple:jopt-simple:jar:4.6:compile -[INFO] org.apache.arrow:arrow-format:jar:5.0.0:compile -[INFO] org.apache.arrow:arrow-memory-core:jar:5.0.0:compile -[INFO] org.apache.arrow:arrow-memory-netty:jar:5.0.0:runtime -[INFO] org.apache.arrow:arrow-vector:jar:5.0.0:compile -[INFO] org.apache.avro:avro:jar:1.10.2:compile -[INFO] org.apache.commons:commons-compress:jar:1.20:compile -[INFO] org.apache.commons:commons-math3:jar:3.2:compile -[INFO] org.apache.httpcomponents:httpclient:jar:4.5.13:compile -[INFO] org.apache.httpcomponents:httpcore:jar:4.4.14:compile -[INFO] org.checkerframework:checker-compat-qual:jar:2.5.5:compile -[INFO] org.checkerframework:checker-qual:jar:3.8.0:compile -[INFO] org.codehaus.mojo:animal-sniffer-annotations:jar:1.20:runtime -[INFO] org.conscrypt:conscrypt-openjdk-uber:jar:2.5.1:compile -[INFO] org.json:json:jar:20200518:compile -[INFO] org.openjdk.jmh:jmh-core:jar:1.32:compile -[INFO] org.openjdk.jmh:jmh-generator-annprocess:jar:1.32:compile -[INFO] org.slf4j:slf4j-api:jar:1.7.30:compile -[INFO] org.threeten:threetenbp:jar:1.5.1:compile From 4d0c5d2f3c5847d86ec1100a47b47e9569aef9b9 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 30 Sep 2021 09:11:07 +0530 Subject: [PATCH 027/277] Updated gitignore --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index e9a78d1ef..1c7a7e78f 100644 --- a/.gitignore +++ b/.gitignore @@ -9,5 +9,5 @@ __pycache__ .diff.txt .new-list.txt .org-list.txt -.flattened-pom.xml -SimpleBenchmarkApp/src/main/java/com/google/cloud/App.java \ No newline at end of file +SimpleBenchmarkApp/src/main/java/com/google/cloud/App.java +.flattened-pom.xml \ No newline at end of file From 1f17ab05a604a4476e38089cb60bc41bc0cd8964 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 1 Oct 2021 14:16:56 +0530 Subject: [PATCH 028/277] updated methods --- .../cloud/bigquery/AbstractJdbcResultSet.java | 21 ++++++------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/AbstractJdbcResultSet.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/AbstractJdbcResultSet.java index 256bbecf8..415350fe4 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/AbstractJdbcResultSet.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/AbstractJdbcResultSet.java @@ -62,6 +62,12 @@ public byte getByte(int columnIndex) throws SQLException { throw new RuntimeException("Not implemented"); } + @Override + public byte getByte(String column) throws SQLException { + // TODO: Implement the logic + throw new RuntimeException("Not implemented"); + } + @Override public short getShort(int columnIndex) throws SQLException { // TODO: Implement the logic @@ -932,21 +938,6 @@ public short getShort(String columnLabel) throws SQLException { throw new SQLFeatureNotSupportedException(); } - @Override - public byte[] getBytes(String columnLabel) throws SQLException { - throw new SQLFeatureNotSupportedException(); - } - - @Override - public Date getDate(String columnLabel) throws SQLException { - throw new SQLFeatureNotSupportedException(); - } - - @Override - public Time getTime(String columnLabel) throws SQLException { - throw new SQLFeatureNotSupportedException(); - } - @Override public InputStream getAsciiStream(String columnLabel) throws SQLException { throw new SQLFeatureNotSupportedException(); From 75f9a529c17cd9b7c8e9713c1df17ee26f0fb16f Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 1 Oct 2021 14:27:22 +0530 Subject: [PATCH 029/277] Fixed implementation and added more methods for BQ datatypes --- .../cloud/bigquery/BigQueryResultSetImpl.java | 118 +++++++++++------- 1 file changed, 76 insertions(+), 42 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index 575402364..ec8353646 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -18,8 +18,10 @@ import com.google.api.services.bigquery.model.TableRow; import java.math.BigDecimal; +import java.sql.Date; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Time; import java.sql.Timestamp; import java.util.concurrent.BlockingQueue; @@ -119,79 +121,111 @@ public int getInt(String fieldName) throws SQLException { @Override public long getLong(String fieldName) throws SQLException { - Object value = getObject(fieldName); - if (value == null) { + if (fieldName == null) { throw new SQLException("fieldName can't be null"); - } else if (!(value instanceof Long)) { - throw new SQLException("value not in instance of Long"); - } else { - return Long.parseLong(String.valueOf(value)); } + if (cursor == null) { + return 0l; // the column value; if the value is SQL NULL, the value returned is 0 as per + // java.sql.ResultSet definition + } else if (cursor instanceof FieldValueList) { + return ((FieldValueList) cursor).get(fieldName).getNumericValue().longValue(); + } + return 0; // TODO: Implementation for Arrow } @Override public double getDouble(String fieldName) throws SQLException { - Object value = getObject(fieldName); - if (value == null) { + if (fieldName == null) { throw new SQLException("fieldName can't be null"); - } else if (!(value instanceof Double)) { - throw new SQLException("value not in instance of Double"); - } else { - return Double.parseDouble(String.valueOf(value)); } + if (cursor == null) { + return 0d; // the column value; if the value is SQL NULL, the value returned is 0 as per + // java.sql.ResultSet definition + } else if (cursor instanceof FieldValueList) { + return ((FieldValueList) cursor).get(fieldName).getNumericValue().doubleValue(); + } + return 0; // TODO: Implementation for Arrow } @Override public BigDecimal getBigDecimal(String fieldName) throws SQLException { - Object value = getObject(fieldName); - if (value == null) { + if (fieldName == null) { throw new SQLException("fieldName can't be null"); - } else if (!(value instanceof Long - || value instanceof Double - || value instanceof BigDecimal - || value instanceof String)) { - throw new SQLException("value cannot be converted to BigDecimal"); - } else { - return new BigDecimal(String.valueOf(value)); } + if (cursor == null) { + return null; // the column value (full precision); if the value is SQL NULL, the value + // returned is null in the Java programming language. as per + // java.sql.ResultSet definition + } else if (cursor instanceof FieldValueList) { + return BigDecimal.valueOf( + ((FieldValueList) cursor).get(fieldName).getNumericValue().doubleValue()); + } + return null; // TODO: Implementation for Arrow } @Override public boolean getBoolean(String fieldName) throws SQLException { - Object value = getObject(fieldName); - if (value == null) { + if (fieldName == null) { throw new SQLException("fieldName can't be null"); - } else if (!(value instanceof Boolean || (value instanceof String))) { - throw new SQLException("value not in instance of Boolean"); - } else { - return Boolean.parseBoolean(String.valueOf(value)); } + if (cursor == null) { + return false; // if the value is SQL NULL, the value returned is false + } else if (cursor instanceof FieldValueList) { + return ((FieldValueList) cursor).get(fieldName).getBooleanValue(); + } + return false; // TODO: Implementation for Arrow } @Override - public byte getByte(String fieldName) throws SQLException { - Object value = getObject(fieldName); - if (value == null) { + public byte[] getBytes(String fieldName) throws SQLException { + if (fieldName == null) { throw new SQLException("fieldName can't be null"); - } else if (!(value instanceof Byte || (value instanceof String))) { - throw new SQLException("value not in instance of Boolean"); - } else { - return Byte.parseByte(String.valueOf(value)); } + if (cursor == null) { + return null; // if the value is SQL NULL, the value returned is null + } else if (cursor instanceof FieldValueList) { + return ((FieldValueList) cursor).get(fieldName).getBytesValue(); + } + return null; // TODO: Implementation for Arrow } @Override public Timestamp getTimestamp(String fieldName) throws SQLException { - Object value = getObject(fieldName); - if (value == null) { + if (fieldName == null) { + throw new SQLException("fieldName can't be null"); + } + if (cursor == null) { + return null; // if the value is SQL NULL, the value returned is null + } else if (cursor instanceof FieldValueList) { + return new Timestamp(((FieldValueList) cursor).get(fieldName).getTimestampValue()); + } + return null; // TODO: Implementation for Arrow + } + + @Override + public Time getTime(String fieldName) throws SQLException { + if (fieldName == null) { + throw new SQLException("fieldName can't be null"); + } + if (cursor == null) { + return null; // if the value is SQL NULL, the value returned is null + } else if (cursor instanceof FieldValueList) { + return new Time(((FieldValueList) cursor).get(fieldName).getTimestampValue()); + } + return null; // TODO: Implementation for Arrow + } + + @Override + public Date getDate(String fieldName) throws SQLException { + if (fieldName == null) { throw new SQLException("fieldName can't be null"); - } else if (!(value instanceof Long - || value instanceof Timestamp - || value instanceof String)) { - throw new SQLException("value cannot be converted to Timestamp"); - } else { - return Timestamp.valueOf(String.valueOf(value)); } + if (cursor == null) { + return null; // if the value is SQL NULL, the value returned is null + } else if (cursor instanceof FieldValueList) { + return new Date(((FieldValueList) cursor).get(fieldName).getTimestampValue()); + } + return null; // TODO: Implementation for Arrow } } } From c7f249716d05bcc6075c9ad1089c3610104056f9 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 8 Oct 2021 19:26:40 +0530 Subject: [PATCH 030/277] Modified testExecuteQuerySinglePageTableRow to cover all the 10 columns --- .../cloud/bigquery/it/ITBigQueryTest.java | 82 +++++++++++++++++-- 1 file changed, 73 insertions(+), 9 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 41671246c..62f83ccfb 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -132,6 +132,8 @@ import java.nio.file.FileSystems; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Time; +import java.time.LocalTime; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -316,6 +318,40 @@ public class ITBigQueryTest { Field.newBuilder("BooleanField", LegacySQLTypeName.BOOLEAN) .setMode(Field.Mode.NULLABLE) .build()); + + private static final Schema BQ_RESULTSET_SCHEMA = + Schema.of( + Field.newBuilder("StringField", LegacySQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("BigNumericField", LegacySQLTypeName.BIGNUMERIC) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("BooleanField", LegacySQLTypeName.BOOLEAN) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("BytesField", LegacySQLTypeName.BYTES) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("IntegerField", LegacySQLTypeName.INTEGER) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("TimestampField", LegacySQLTypeName.TIMESTAMP) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("Date_Field", LegacySQLTypeName.DATE) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("FloatField", LegacySQLTypeName.FLOAT) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("NumericField", LegacySQLTypeName.NUMERIC) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("TimeField", LegacySQLTypeName.TIME) + .setMode(Field.Mode.NULLABLE) + .build()); + private static final Schema QUERY_RESULT_SCHEMA_BIGNUMERIC = Schema.of( Field.newBuilder("TimestampField", LegacySQLTypeName.TIMESTAMP) @@ -1959,20 +1995,48 @@ public void testQueryExternalHivePartitioningOptionCustomLayout() throws Interru @Test public void testExecuteQuerySinglePageTableRow() throws SQLException { String query = - "SELECT TimestampField, StringField, BooleanField FROM " + TABLE_ID_FASTQUERY.getTable(); + "select StringField, BigNumericField, BooleanField, BytesField, IntegerField, TimestampField, Date_Field, FloatField, NumericField, TimeField " + + "from fastquery_testing_table_bqresultset order by TimestampField"; ConnectionSettings connectionSettings = - ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); + ConnectionSettings.newBuilder() + .setDefaultDataset(DatasetId.of("bigquery_test_dataset")) + .build(); Connection connection = bigquery.createConnection(connectionSettings); BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); ResultSet rs = bigQueryResultSet.getResultSet(); - // while (rs.next()) { - // System.out.println(rs.getString("TimestampField")); - // } - assertEquals(QUERY_RESULT_SCHEMA, bigQueryResultSet.getSchema()); + Schema sc = bigQueryResultSet.getSchema(); + + assertEquals(BQ_RESULTSET_SCHEMA, sc); // match the schema + assertEquals(2, bigQueryResultSet.getTotalRows()); - assertTrue(rs.next()); - assertTrue(rs.next()); - assertFalse(rs.next()); // it should return false for the third call as the table has 2 rows + + assertTrue(rs.next()); // first row + // checking for the null or 0 column values + assertNull(rs.getString("StringField")); + assertTrue(rs.getDouble("BigNumericField") == 0.0d); + assertFalse(rs.getBoolean("BooleanField")); + assertNull(rs.getBytes("BytesField")); + assertEquals(rs.getInt("IntegerField"), 0); + assertNull(rs.getTimestamp("TimestampField")); + assertNull(rs.getDate("Date_Field")); + assertTrue(rs.getDouble("FloatField") == 0.0d); + assertTrue(rs.getDouble("NumericField") == 0.0d); + assertNull(rs.getTime("TimeField")); + + assertTrue(rs.next()); // second row + // second row is non null, comparing the values + assertEquals(rs.getString("StringField"), "StringValue1"); + assertTrue(rs.getDouble("BigNumericField") == 10.0d); + assertFalse(rs.getBoolean("BooleanField")); + assertNotNull(rs.getBytes("BytesField")); + assertEquals(rs.getInt("IntegerField"), 1); + assertEquals(rs.getTimestamp("TimestampField").toString(), "2018-08-19 17:41:35.22"); + assertEquals(rs.getDate("Date_Field"), java.sql.Date.valueOf("2018-08-19")); + assertTrue(rs.getDouble("FloatField") == 10.1d); + assertTrue(rs.getDouble("NumericField") == 100.0d); + assertEquals(rs.getTime("TimeField"), Time.valueOf(LocalTime.of(12, 11, 35, 000000))); + + assertFalse(rs.next()); // no third row in the table } @Test From 8a2f5333dc3f3c0f86f517bcff63345e6904c0ce Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 8 Oct 2021 19:27:20 +0530 Subject: [PATCH 031/277] Fixed the implementation of RS methods --- .../cloud/bigquery/BigQueryResultSetImpl.java | 66 +++++++++++++++---- 1 file changed, 52 insertions(+), 14 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index ec8353646..978e025ba 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -23,6 +23,7 @@ import java.sql.SQLException; import java.sql.Time; import java.sql.Timestamp; +import java.time.LocalTime; import java.util.concurrent.BlockingQueue; // TODO: This implementation deals with the JSON response. We can have respective implementations @@ -100,7 +101,8 @@ public String getString(String fieldName) throws SQLException { if (cursor == null) { return null; } else if (cursor instanceof FieldValueList) { - return ((FieldValueList) cursor).get(fieldName).getStringValue(); + FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); + return fieldValue.getValue() == null ? null : fieldValue.getStringValue(); } return null; // TODO: Implementation for Arrow } @@ -114,7 +116,8 @@ public int getInt(String fieldName) throws SQLException { return 0; // the column value; if the value is SQL NULL, the value returned is 0 as per // java.sql.ResultSet definition } else if (cursor instanceof FieldValueList) { - return ((FieldValueList) cursor).get(fieldName).getNumericValue().intValue(); + FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); + return fieldValue.getValue() == null ? 0 : fieldValue.getNumericValue().intValue(); } return 0; // TODO: Implementation for Arrow } @@ -125,12 +128,13 @@ public long getLong(String fieldName) throws SQLException { throw new SQLException("fieldName can't be null"); } if (cursor == null) { - return 0l; // the column value; if the value is SQL NULL, the value returned is 0 as per + return 0L; // the column value; if the value is SQL NULL, the value returned is 0 as per // java.sql.ResultSet definition } else if (cursor instanceof FieldValueList) { - return ((FieldValueList) cursor).get(fieldName).getNumericValue().longValue(); + FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); + return fieldValue.getValue() == null ? 0L : fieldValue.getNumericValue().longValue(); } - return 0; // TODO: Implementation for Arrow + return 0L; // TODO: Implementation for Arrow } @Override @@ -142,9 +146,10 @@ public double getDouble(String fieldName) throws SQLException { return 0d; // the column value; if the value is SQL NULL, the value returned is 0 as per // java.sql.ResultSet definition } else if (cursor instanceof FieldValueList) { - return ((FieldValueList) cursor).get(fieldName).getNumericValue().doubleValue(); + FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); + return fieldValue.getValue() == null ? 0d : fieldValue.getNumericValue().doubleValue(); } - return 0; // TODO: Implementation for Arrow + return 0d; // TODO: Implementation for Arrow } @Override @@ -157,8 +162,10 @@ public BigDecimal getBigDecimal(String fieldName) throws SQLException { // returned is null in the Java programming language. as per // java.sql.ResultSet definition } else if (cursor instanceof FieldValueList) { - return BigDecimal.valueOf( - ((FieldValueList) cursor).get(fieldName).getNumericValue().doubleValue()); + FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); + return fieldValue.getValue() == null + ? null + : BigDecimal.valueOf(fieldValue.getNumericValue().doubleValue()); } return null; // TODO: Implementation for Arrow } @@ -171,7 +178,8 @@ public boolean getBoolean(String fieldName) throws SQLException { if (cursor == null) { return false; // if the value is SQL NULL, the value returned is false } else if (cursor instanceof FieldValueList) { - return ((FieldValueList) cursor).get(fieldName).getBooleanValue(); + FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); + return fieldValue.getValue() != null && fieldValue.getBooleanValue(); } return false; // TODO: Implementation for Arrow } @@ -184,7 +192,8 @@ public byte[] getBytes(String fieldName) throws SQLException { if (cursor == null) { return null; // if the value is SQL NULL, the value returned is null } else if (cursor instanceof FieldValueList) { - return ((FieldValueList) cursor).get(fieldName).getBytesValue(); + FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); + return fieldValue.getValue() == null ? null : fieldValue.getBytesValue(); } return null; // TODO: Implementation for Arrow } @@ -197,7 +206,13 @@ public Timestamp getTimestamp(String fieldName) throws SQLException { if (cursor == null) { return null; // if the value is SQL NULL, the value returned is null } else if (cursor instanceof FieldValueList) { - return new Timestamp(((FieldValueList) cursor).get(fieldName).getTimestampValue()); + FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); + return fieldValue.getValue() == null + ? null + : new Timestamp( + fieldValue.getTimestampValue() + / 1000); // getTimestampValue returns time in microseconds, and TimeStamp + // expects it in millis } return null; // TODO: Implementation for Arrow } @@ -210,7 +225,29 @@ public Time getTime(String fieldName) throws SQLException { if (cursor == null) { return null; // if the value is SQL NULL, the value returned is null } else if (cursor instanceof FieldValueList) { - return new Time(((FieldValueList) cursor).get(fieldName).getTimestampValue()); + FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); + if (fieldValue.getValue() != null) { + // Time ranges from 00:00:00 to 23:59:59.99999. in BigQuery. Parsing it to java.sql.Time + String strTime = fieldValue.getStringValue(); + String[] timeSplt = strTime.split(":"); + if (timeSplt.length != 3) { + throw new SQLException("Can not parse the value " + strTime + " to java.sql.Time"); + } + int hr = Integer.parseInt(timeSplt[0]); + int min = Integer.parseInt(timeSplt[1]); + int sec = 0, nanoSec = 0; + if (timeSplt[2].contains(".")) { + String[] secSplt = timeSplt[2].split("\\."); + sec = Integer.parseInt(secSplt[0]); + nanoSec = Integer.parseInt(secSplt[1]); + } else { + sec = Integer.parseInt(timeSplt[2]); + } + // LocalTime.of() + return Time.valueOf(LocalTime.of(hr, min, sec, nanoSec)); + } else { + return null; + } } return null; // TODO: Implementation for Arrow } @@ -223,7 +260,8 @@ public Date getDate(String fieldName) throws SQLException { if (cursor == null) { return null; // if the value is SQL NULL, the value returned is null } else if (cursor instanceof FieldValueList) { - return new Date(((FieldValueList) cursor).get(fieldName).getTimestampValue()); + FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); + return fieldValue.getValue() == null ? null : Date.valueOf(fieldValue.getStringValue()); } return null; // TODO: Implementation for Arrow } From bee2ef4fd7e9e13bdad587ff8040633927e91756 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Mon, 11 Oct 2021 17:44:48 +0530 Subject: [PATCH 032/277] Execute query test case fix --- .../cloud/bigquery/it/ITBigQueryTest.java | 231 ++++++++++++++++-- 1 file changed, 207 insertions(+), 24 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 62f83ccfb..b3da80350 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -167,6 +167,7 @@ public class ITBigQueryTest { private static final String OTHER_DATASET = RemoteBigQueryHelper.generateDatasetName(); private static final String MODEL_DATASET = RemoteBigQueryHelper.generateDatasetName(); private static final String ROUTINE_DATASET = RemoteBigQueryHelper.generateDatasetName(); + private static final String BQ_RESULTSET_DATASET = RemoteBigQueryHelper.generateDatasetName(); private static final String PROJECT_ID = ServiceOptions.getDefaultProjectId(); private static final String RANDOM_ID = UUID.randomUUID().toString().substring(0, 8); private static final String CLOUD_SAMPLES_DATA = @@ -180,6 +181,26 @@ public class ITBigQueryTest { .setMode(Field.Mode.NULLABLE) .setDescription("TimestampDescription") .build(); + private static final Field TIME_FIELD_SCHEMA = + Field.newBuilder("TimeField", LegacySQLTypeName.TIME) + .setMode(Field.Mode.NULLABLE) + .setDescription("TimeDescription") + .build(); + private static final Field DATE_FIELD_SCHEMA = + Field.newBuilder("DateField", LegacySQLTypeName.DATE) + .setMode(Field.Mode.NULLABLE) + .setDescription("DateDescription") + .build(); + private static final Field DATE_TIME_FIELD_SCHEMA = + Field.newBuilder("DateTimeField", LegacySQLTypeName.DATETIME) + .setMode(Field.Mode.NULLABLE) + .setDescription("DateTimeDescription") + .build(); + private static final Field STRUCT_FIELD_SCHEMA = + Field.newBuilder("StructField", LegacySQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .setDescription("StructDescription") + .build(); private static final Field STRING_FIELD_SCHEMA = Field.newBuilder("StringField", LegacySQLTypeName.STRING) .setMode(Field.Mode.NULLABLE) @@ -275,6 +296,30 @@ public class ITBigQueryTest { BIGNUMERIC_FIELD_SCHEMA3, BIGNUMERIC_FIELD_SCHEMA4); + private static final Schema BQ_RESULTSET_SCHEMA = + Schema.of( + TIMESTAMP_FIELD_SCHEMA, + STRING_FIELD_SCHEMA, + INTEGER_ARRAY_FIELD_SCHEMA, + BOOLEAN_FIELD_SCHEMA, + BYTES_FIELD_SCHEMA, + RECORD_FIELD_SCHEMA, + INTEGER_FIELD_SCHEMA, + FLOAT_FIELD_SCHEMA, + GEOGRAPHY_FIELD_SCHEMA, + NUMERIC_FIELD_SCHEMA, + BIGNUMERIC_FIELD_SCHEMA, + BIGNUMERIC_FIELD_SCHEMA1, + BIGNUMERIC_FIELD_SCHEMA2, + BIGNUMERIC_FIELD_SCHEMA3, + BIGNUMERIC_FIELD_SCHEMA4, + TIME_FIELD_SCHEMA, + DATE_FIELD_SCHEMA, + DATE_TIME_FIELD_SCHEMA, + STRUCT_FIELD_SCHEMA + ); + + private static final Field DDL_TIMESTAMP_FIELD_SCHEMA = Field.newBuilder("TimestampField", LegacySQLTypeName.TIMESTAMP) .setDescription("TimestampDescription") @@ -319,11 +364,29 @@ public class ITBigQueryTest { .setMode(Field.Mode.NULLABLE) .build()); - private static final Schema BQ_RESULTSET_SCHEMA = + /* + private static final Schema BQ_RESULTSET_EXPECTED_SCHEMA = + Schema.of( + STRING_FIELD_SCHEMA, + BIGNUMERIC_FIELD_SCHEMA, + BOOLEAN_FIELD_SCHEMA, + BYTES_FIELD_SCHEMA, + INTEGER_FIELD_SCHEMA, + TIMESTAMP_FIELD_SCHEMA, + FLOAT_FIELD_SCHEMA, + NUMERIC_FIELD_SCHEMA, + TIME_FIELD_SCHEMA, + DATE_FIELD_SCHEMA, + DATE_TIME_FIELD_SCHEMA, + STRUCT_FIELD_SCHEMA, + INTEGER_ARRAY_FIELD_SCHEMA + ); + */ + private static final Schema BQ_RESULTSET_EXPECTED_SCHEMA = Schema.of( Field.newBuilder("StringField", LegacySQLTypeName.STRING) .setMode(Field.Mode.NULLABLE) - .build(), + .build() , Field.newBuilder("BigNumericField", LegacySQLTypeName.BIGNUMERIC) .setMode(Field.Mode.NULLABLE) .build(), @@ -339,18 +402,31 @@ public class ITBigQueryTest { Field.newBuilder("TimestampField", LegacySQLTypeName.TIMESTAMP) .setMode(Field.Mode.NULLABLE) .build(), + Field.newBuilder("FloatField", LegacySQLTypeName.FLOAT) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("NumericField", LegacySQLTypeName.NUMERIC) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("TimeField", LegacySQLTypeName.TIME) + .setMode(Field.Mode.NULLABLE) + .build(), Field.newBuilder("Date_Field", LegacySQLTypeName.DATE) .setMode(Field.Mode.NULLABLE) .build(), - Field.newBuilder("FloatField", LegacySQLTypeName.FLOAT) + Field.newBuilder("Date_Time_Field", LegacySQLTypeName.DATETIME) .setMode(Field.Mode.NULLABLE) .build(), - Field.newBuilder("NumericField", LegacySQLTypeName.NUMERIC) - .setMode(Field.Mode.NULLABLE) - .build(), - Field.newBuilder("TimeField", LegacySQLTypeName.TIME) - .setMode(Field.Mode.NULLABLE) - .build()); + Field.newBuilder("Struct_Field", LegacySQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("IntegerArrayField", LegacySQLTypeName.DATE) + .setMode(Field.Mode.NULLABLE) + .build() + + + + ); private static final Schema QUERY_RESULT_SCHEMA_BIGNUMERIC = Schema.of( @@ -396,6 +472,7 @@ public class ITBigQueryTest { private static final String LOAD_FILE = "load.csv"; private static final String LOAD_FILE_LARGE = "load_large.csv"; private static final String JSON_LOAD_FILE = "load.json"; + private static final String JSON_LOAD_FILE_BQ_RESULTSET = "load_bq_resultset.json"; private static final String JSON_LOAD_FILE_SIMPLE = "load_simple.json"; private static final String EXTRACT_FILE = "extract.csv"; private static final String EXTRACT_MODEL_FILE = "extract_model.csv"; @@ -404,7 +481,9 @@ public class ITBigQueryTest { private static final TableId TABLE_ID_DDL = TableId.of(DATASET, "ddl_testing_table"); private static final TableId TABLE_ID_FASTQUERY = TableId.of(DATASET, "fastquery_testing_table"); private static final TableId TABLE_ID_LARGE = TableId.of(DATASET, "large_data_testing_table"); + private static final TableId TABLE_ID_FASTQUERY_BQ_RESULTSET = TableId.of(DATASET, "fastquery_testing_bq_resultset"); private static final String CSV_CONTENT = "StringValue1\nStringValue2\n"; + private static final String JSON_CONTENT = "{" + " \"TimestampField\": \"2014-08-19 07:41:35.220 -05:00\"," @@ -460,6 +539,66 @@ public class ITBigQueryTest { + " \"BigNumericField3\": \"578960446186580977117854925043439539266.34992332820282019728792003956564819967\"," + " \"BigNumericField4\": \"-578960446186580977117854925043439539266.34992332820282019728792003956564819968\"" + "}"; + + private static final String JSON_CONTENT_BQ_RESULTSET = + "{" + + " \"TimestampField\": null," + + " \"StringField\": null," + + " \"IntegerArrayField\": null," + + " \"BooleanField\": null," + + " \"BytesField\": null," + + " \"RecordField\": {" + + " \"TimestampField\": null," + + " \"StringField\": null," + + " \"IntegerArrayField\": null," + + " \"BooleanField\": null," + + " \"BytesField\": null" + + " }," + + " \"IntegerField\": null," + + " \"FloatField\": null," + + " \"GeographyField\": null," + + " \"NumericField\": null," + + " \"BigNumericField\": null," + + " \"BigNumericField1\": null," + + " \"BigNumericField2\": null," + + " \"BigNumericField3\": null," + + " \"BigNumericField4\": null," + + " \"TimeField\": null," + + " \"DateField\": null," + + " \"DateTimeField\": null," + + " \"StructField\": null" + + "}\n" + + "{" + + " \"TimestampField\": \"2018-08-19 12:11:35.220 UTC\"," + + " \"StringField\": \"StringValue1\"," + + " \"IntegerArrayField\": [\"0\", \"1\"]," + + " \"BooleanField\": \"false\"," + + " \"BytesField\": \"" + + BYTES_BASE64 + + "\"," + + " \"RecordField\": {" + + " \"TimestampField\": \"1969-07-20 20:18:04 UTC\"," + + " \"StringField\": null," + + " \"IntegerArrayField\": [\"1\",\"0\"]," + + " \"BooleanField\": \"true\"," + + " \"BytesField\": \"" + + BYTES_BASE64 + + "\"" + + " }," + + " \"IntegerField\": \"1\"," + + " \"FloatField\": \"10.1\"," + + " \"GeographyField\": \"POINT(-122.35022 47.649154)\"," + + " \"NumericField\": \"100\"," + + " \"BigNumericField\": \"0.33333333333333333333333333333333333333\"," + + " \"BigNumericField1\": \"1e-38\"," + + " \"BigNumericField2\": \"-1e38\"," + + " \"BigNumericField3\": \"578960446186580977117854925043439539266.34992332820282019728792003956564819967\"," + + " \"BigNumericField4\": \"-578960446186580977117854925043439539266.34992332820282019728792003956564819968\"," + + " \"TimeField\": \"12:11:35.123456\"," + + " \"DateField\": \"2018-08-19\"," + + " \"DateTimeField\": \"2018-08-19 12:11:35.123456\"," + + " \"StructField\": \"STRUCT('StructVal')\"" + + "}"; private static final String JSON_CONTENT_SIMPLE = "{" + " \"TimestampField\": \"2014-08-19 07:41:35.220 -05:00\"," @@ -511,6 +650,9 @@ public static void beforeClass() throws InterruptedException, IOException { storage.createFrom( BlobInfo.newBuilder(BUCKET, LOAD_FILE_LARGE).setContentType("text/plain").build(), FileSystems.getDefault().getPath("src/test/resources", "QueryTestData.csv")); + storage.create( + BlobInfo.newBuilder(BUCKET, JSON_LOAD_FILE_BQ_RESULTSET).setContentType("application/json").build(), + JSON_CONTENT_BQ_RESULTSET.getBytes(StandardCharsets.UTF_8)); DatasetInfo info = DatasetInfo.newBuilder(DATASET).setDescription(DESCRIPTION).setLabels(LABELS).build(); bigquery.create(info); @@ -544,7 +686,18 @@ public static void beforeClass() throws InterruptedException, IOException { jobFastQuery = jobFastQuery.waitFor(); assertNull(jobFastQuery.getStatus().getError()); - LoadJobConfiguration configurationDDL = + LoadJobConfiguration configFastQueryBQResultset = + LoadJobConfiguration.newBuilder( + TABLE_ID_FASTQUERY_BQ_RESULTSET, "gs://" + BUCKET + "/" + JSON_LOAD_FILE_BQ_RESULTSET, FormatOptions.json()) + .setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) + .setSchema(BQ_RESULTSET_SCHEMA) + .setLabels(labels) + .build(); + Job jobFastQueryBQResultSet = bigquery.create(JobInfo.of(configFastQueryBQResultset)); + jobFastQueryBQResultSet = jobFastQueryBQResultSet.waitFor(); + assertNull(jobFastQueryBQResultSet.getStatus().getError()); + + LoadJobConfiguration configurationDDL = LoadJobConfiguration.newBuilder( TABLE_ID_DDL, "gs://" + BUCKET + "/" + JSON_LOAD_FILE_SIMPLE, FormatOptions.json()) .setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) @@ -1995,20 +2148,38 @@ public void testQueryExternalHivePartitioningOptionCustomLayout() throws Interru @Test public void testExecuteQuerySinglePageTableRow() throws SQLException { String query = - "select StringField, BigNumericField, BooleanField, BytesField, IntegerField, TimestampField, Date_Field, FloatField, NumericField, TimeField " - + "from fastquery_testing_table_bqresultset order by TimestampField"; + "select StringField, BigNumericField, BooleanField, BytesField, IntegerField, TimestampField, FloatField, " + + "NumericField, TimeField, DateField, DateTimeField, StructField, IntegerArrayField from "+ TABLE_ID_FASTQUERY_BQ_RESULTSET.getTable() + + " order by TimestampField"; ConnectionSettings connectionSettings = ConnectionSettings.newBuilder() - .setDefaultDataset(DatasetId.of("bigquery_test_dataset")) + .setDefaultDataset(DatasetId.of(DATASET)) .build(); Connection connection = bigquery.createConnection(connectionSettings); BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); ResultSet rs = bigQueryResultSet.getResultSet(); Schema sc = bigQueryResultSet.getSchema(); - assertEquals(BQ_RESULTSET_SCHEMA, sc); // match the schema - - assertEquals(2, bigQueryResultSet.getTotalRows()); + assertEquals(BQ_RESULTSET_EXPECTED_SCHEMA, sc); // match the schema + + assertEquals(3, bigQueryResultSet.getTotalRows()); + +/* while (rs.next()){ + System.out.println(rs.getString("StringField")); + System.out.println(rs.getDouble("BigNumericField")); + System.out.println(rs.getBoolean("BooleanField")); + System.out.println(rs.getBytes("BytesField")); + System.out.println(rs.getInt("IntegerField")); + System.out.println(rs.getTimestamp("TimestampField")); + System.out.println(rs.getDate("DateField")); + System.out.println(rs.getDouble("FloatField")); + System.out.println(rs.getDouble("NumericField")); + System.out.println(rs.getTime("TimeField")); + System.out.println(rs.getString("DateTimeField")); + System.out.println(rs.getString("StructField")); + System.out.println(rs.getString("IntegerArrayField")); + System.out.println("\n\n>\n\n"); + }*/ assertTrue(rs.next()); // first row // checking for the null or 0 column values @@ -2018,25 +2189,37 @@ public void testExecuteQuerySinglePageTableRow() throws SQLException { assertNull(rs.getBytes("BytesField")); assertEquals(rs.getInt("IntegerField"), 0); assertNull(rs.getTimestamp("TimestampField")); - assertNull(rs.getDate("Date_Field")); + assertNull(rs.getDate("DateField")); assertTrue(rs.getDouble("FloatField") == 0.0d); assertTrue(rs.getDouble("NumericField") == 0.0d); assertNull(rs.getTime("TimeField")); + assertNull(rs.getString("DateTimeField")); + assertNull(rs.getString("StructField")); + assertNull(rs.getString("IntegerArrayField")); assertTrue(rs.next()); // second row // second row is non null, comparing the values - assertEquals(rs.getString("StringField"), "StringValue1"); - assertTrue(rs.getDouble("BigNumericField") == 10.0d); + assertEquals("StringValue1", rs.getString("StringField")); + assertTrue(rs.getDouble("BigNumericField") == 0.3333333333333333d); assertFalse(rs.getBoolean("BooleanField")); assertNotNull(rs.getBytes("BytesField")); - assertEquals(rs.getInt("IntegerField"), 1); - assertEquals(rs.getTimestamp("TimestampField").toString(), "2018-08-19 17:41:35.22"); - assertEquals(rs.getDate("Date_Field"), java.sql.Date.valueOf("2018-08-19")); + assertEquals(1, rs.getInt("IntegerField")); + /* +org.junit.ComparisonFailure: +Expected :2018-08-19 17:41:35.123456 +Actual :2018-08-19 17:41:35.22 + */ + // assertEquals("2018-08-19 17:41:35.123456", rs.getTimestamp("TimestampField").toString()); + assertEquals( java.sql.Date.valueOf("2018-08-19"), rs.getDate("DateField")); assertTrue(rs.getDouble("FloatField") == 10.1d); assertTrue(rs.getDouble("NumericField") == 100.0d); - assertEquals(rs.getTime("TimeField"), Time.valueOf(LocalTime.of(12, 11, 35, 000000))); + assertEquals(Time.valueOf(LocalTime.of(12, 11, 35, 123456)), rs.getTime("TimeField")); + assertEquals("2018-08-19T12:11:35.123456", rs.getString("DateTimeField")); + assertEquals("STRUCT('StructVal')", rs.getString("StructField")); + assertEquals("0", rs.getString("IntegerArrayField")); - assertFalse(rs.next()); // no third row in the table + assertTrue(rs.next()); //third row + assertFalse(rs.next()); // no 4th row in the table } @Test From 3d908cec76315ed23f347a39770ce6a3a2c34a02 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Mon, 11 Oct 2021 18:49:34 +0530 Subject: [PATCH 033/277] Fixed testExecuteQuerySinglePageTableRow testcase --- .../cloud/bigquery/it/ITBigQueryTest.java | 210 ++++++++---------- 1 file changed, 94 insertions(+), 116 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index b3da80350..53fc03474 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -211,6 +211,11 @@ public class ITBigQueryTest { .setMode(Field.Mode.REPEATED) .setDescription("IntegerArrayDescription") .build(); + private static final Field STRING_ARRAY_FIELD_SCHEMA = + Field.newBuilder("StringArrayField", LegacySQLTypeName.STRING) + .setMode(Field.Mode.REPEATED) + .setDescription("StringArrayDescription") + .build(); private static final Field BOOLEAN_FIELD_SCHEMA = Field.newBuilder("BooleanField", LegacySQLTypeName.BOOLEAN) .setMode(Field.Mode.NULLABLE) @@ -313,12 +318,11 @@ public class ITBigQueryTest { BIGNUMERIC_FIELD_SCHEMA2, BIGNUMERIC_FIELD_SCHEMA3, BIGNUMERIC_FIELD_SCHEMA4, - TIME_FIELD_SCHEMA, - DATE_FIELD_SCHEMA, - DATE_TIME_FIELD_SCHEMA, - STRUCT_FIELD_SCHEMA - ); - + TIME_FIELD_SCHEMA, + DATE_FIELD_SCHEMA, + DATE_TIME_FIELD_SCHEMA, + STRUCT_FIELD_SCHEMA, + STRING_ARRAY_FIELD_SCHEMA); private static final Field DDL_TIMESTAMP_FIELD_SCHEMA = Field.newBuilder("TimestampField", LegacySQLTypeName.TIMESTAMP) @@ -364,29 +368,11 @@ public class ITBigQueryTest { .setMode(Field.Mode.NULLABLE) .build()); - /* - private static final Schema BQ_RESULTSET_EXPECTED_SCHEMA = - Schema.of( - STRING_FIELD_SCHEMA, - BIGNUMERIC_FIELD_SCHEMA, - BOOLEAN_FIELD_SCHEMA, - BYTES_FIELD_SCHEMA, - INTEGER_FIELD_SCHEMA, - TIMESTAMP_FIELD_SCHEMA, - FLOAT_FIELD_SCHEMA, - NUMERIC_FIELD_SCHEMA, - TIME_FIELD_SCHEMA, - DATE_FIELD_SCHEMA, - DATE_TIME_FIELD_SCHEMA, - STRUCT_FIELD_SCHEMA, - INTEGER_ARRAY_FIELD_SCHEMA - ); - */ private static final Schema BQ_RESULTSET_EXPECTED_SCHEMA = Schema.of( Field.newBuilder("StringField", LegacySQLTypeName.STRING) .setMode(Field.Mode.NULLABLE) - .build() , + .build(), Field.newBuilder("BigNumericField", LegacySQLTypeName.BIGNUMERIC) .setMode(Field.Mode.NULLABLE) .build(), @@ -402,31 +388,30 @@ public class ITBigQueryTest { Field.newBuilder("TimestampField", LegacySQLTypeName.TIMESTAMP) .setMode(Field.Mode.NULLABLE) .build(), - Field.newBuilder("FloatField", LegacySQLTypeName.FLOAT) - .setMode(Field.Mode.NULLABLE) - .build(), - Field.newBuilder("NumericField", LegacySQLTypeName.NUMERIC) - .setMode(Field.Mode.NULLABLE) - .build(), - Field.newBuilder("TimeField", LegacySQLTypeName.TIME) - .setMode(Field.Mode.NULLABLE) - .build(), - Field.newBuilder("Date_Field", LegacySQLTypeName.DATE) + Field.newBuilder("FloatField", LegacySQLTypeName.FLOAT) .setMode(Field.Mode.NULLABLE) .build(), - Field.newBuilder("Date_Time_Field", LegacySQLTypeName.DATETIME) + Field.newBuilder("NumericField", LegacySQLTypeName.NUMERIC) .setMode(Field.Mode.NULLABLE) .build(), - Field.newBuilder("Struct_Field", LegacySQLTypeName.STRING) - .setMode(Field.Mode.NULLABLE) - .build(), - Field.newBuilder("IntegerArrayField", LegacySQLTypeName.DATE) - .setMode(Field.Mode.NULLABLE) - .build() - - - - ); + Field.newBuilder("TimeField", LegacySQLTypeName.TIME) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("DateField", LegacySQLTypeName.DATE) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("DateTimeField", LegacySQLTypeName.DATETIME) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("StructField", LegacySQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("StringArrayField", LegacySQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("GeographyField", LegacySQLTypeName.GEOGRAPHY) + .setMode(Field.Mode.NULLABLE) + .build()); private static final Schema QUERY_RESULT_SCHEMA_BIGNUMERIC = Schema.of( @@ -481,7 +466,8 @@ public class ITBigQueryTest { private static final TableId TABLE_ID_DDL = TableId.of(DATASET, "ddl_testing_table"); private static final TableId TABLE_ID_FASTQUERY = TableId.of(DATASET, "fastquery_testing_table"); private static final TableId TABLE_ID_LARGE = TableId.of(DATASET, "large_data_testing_table"); - private static final TableId TABLE_ID_FASTQUERY_BQ_RESULTSET = TableId.of(DATASET, "fastquery_testing_bq_resultset"); + private static final TableId TABLE_ID_FASTQUERY_BQ_RESULTSET = + TableId.of(DATASET, "fastquery_testing_bq_resultset"); private static final String CSV_CONTENT = "StringValue1\nStringValue2\n"; private static final String JSON_CONTENT = @@ -541,34 +527,35 @@ public class ITBigQueryTest { + "}"; private static final String JSON_CONTENT_BQ_RESULTSET = - "{" - + " \"TimestampField\": null," - + " \"StringField\": null," - + " \"IntegerArrayField\": null," - + " \"BooleanField\": null," - + " \"BytesField\": null," - + " \"RecordField\": {" - + " \"TimestampField\": null," - + " \"StringField\": null," - + " \"IntegerArrayField\": null," - + " \"BooleanField\": null," - + " \"BytesField\": null" - + " }," - + " \"IntegerField\": null," - + " \"FloatField\": null," - + " \"GeographyField\": null," - + " \"NumericField\": null," - + " \"BigNumericField\": null," - + " \"BigNumericField1\": null," - + " \"BigNumericField2\": null," - + " \"BigNumericField3\": null," - + " \"BigNumericField4\": null," - + " \"TimeField\": null," - + " \"DateField\": null," - + " \"DateTimeField\": null," - + " \"StructField\": null" - + "}\n" - + "{" + "{" + + " \"TimestampField\": null," + + " \"StringField\": null," + + " \"IntegerArrayField\": null," + + " \"BooleanField\": null," + + " \"BytesField\": null," + + " \"RecordField\": {" + + " \"TimestampField\": null," + + " \"StringField\": null," + + " \"IntegerArrayField\": null," + + " \"BooleanField\": null," + + " \"BytesField\": null" + + " }," + + " \"IntegerField\": null," + + " \"FloatField\": null," + + " \"GeographyField\": null," + + " \"NumericField\": null," + + " \"BigNumericField\": null," + + " \"BigNumericField1\": null," + + " \"BigNumericField2\": null," + + " \"BigNumericField3\": null," + + " \"BigNumericField4\": null," + + " \"TimeField\": null," + + " \"DateField\": null," + + " \"DateTimeField\": null," + + " \"StructField\": null," + + " \"StringArrayField\": null" + + "}\n" + + "{" + " \"TimestampField\": \"2018-08-19 12:11:35.220 UTC\"," + " \"StringField\": \"StringValue1\"," + " \"IntegerArrayField\": [\"0\", \"1\"]," @@ -597,7 +584,8 @@ public class ITBigQueryTest { + " \"TimeField\": \"12:11:35.123456\"," + " \"DateField\": \"2018-08-19\"," + " \"DateTimeField\": \"2018-08-19 12:11:35.123456\"," - + " \"StructField\": \"STRUCT('StructVal')\"" + + " \"StructField\": \"STRUCT('StructVal')\"," + + " \"StringArrayField\": [\"one\", \"two\"]" + "}"; private static final String JSON_CONTENT_SIMPLE = "{" @@ -651,8 +639,10 @@ public static void beforeClass() throws InterruptedException, IOException { BlobInfo.newBuilder(BUCKET, LOAD_FILE_LARGE).setContentType("text/plain").build(), FileSystems.getDefault().getPath("src/test/resources", "QueryTestData.csv")); storage.create( - BlobInfo.newBuilder(BUCKET, JSON_LOAD_FILE_BQ_RESULTSET).setContentType("application/json").build(), - JSON_CONTENT_BQ_RESULTSET.getBytes(StandardCharsets.UTF_8)); + BlobInfo.newBuilder(BUCKET, JSON_LOAD_FILE_BQ_RESULTSET) + .setContentType("application/json") + .build(), + JSON_CONTENT_BQ_RESULTSET.getBytes(StandardCharsets.UTF_8)); DatasetInfo info = DatasetInfo.newBuilder(DATASET).setDescription(DESCRIPTION).setLabels(LABELS).build(); bigquery.create(info); @@ -687,17 +677,19 @@ public static void beforeClass() throws InterruptedException, IOException { assertNull(jobFastQuery.getStatus().getError()); LoadJobConfiguration configFastQueryBQResultset = - LoadJobConfiguration.newBuilder( - TABLE_ID_FASTQUERY_BQ_RESULTSET, "gs://" + BUCKET + "/" + JSON_LOAD_FILE_BQ_RESULTSET, FormatOptions.json()) - .setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) - .setSchema(BQ_RESULTSET_SCHEMA) - .setLabels(labels) - .build(); + LoadJobConfiguration.newBuilder( + TABLE_ID_FASTQUERY_BQ_RESULTSET, + "gs://" + BUCKET + "/" + JSON_LOAD_FILE_BQ_RESULTSET, + FormatOptions.json()) + .setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) + .setSchema(BQ_RESULTSET_SCHEMA) + .setLabels(labels) + .build(); Job jobFastQueryBQResultSet = bigquery.create(JobInfo.of(configFastQueryBQResultset)); jobFastQueryBQResultSet = jobFastQueryBQResultSet.waitFor(); assertNull(jobFastQueryBQResultSet.getStatus().getError()); - LoadJobConfiguration configurationDDL = + LoadJobConfiguration configurationDDL = LoadJobConfiguration.newBuilder( TABLE_ID_DDL, "gs://" + BUCKET + "/" + JSON_LOAD_FILE_SIMPLE, FormatOptions.json()) .setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) @@ -2148,13 +2140,12 @@ public void testQueryExternalHivePartitioningOptionCustomLayout() throws Interru @Test public void testExecuteQuerySinglePageTableRow() throws SQLException { String query = - "select StringField, BigNumericField, BooleanField, BytesField, IntegerField, TimestampField, FloatField, " + - "NumericField, TimeField, DateField, DateTimeField, StructField, IntegerArrayField from "+ TABLE_ID_FASTQUERY_BQ_RESULTSET.getTable() + "select StringField, BigNumericField, BooleanField, BytesField, IntegerField, TimestampField, FloatField, " + + "NumericField, TimeField, DateField, DateTimeField, StructField, StringArrayField , GeographyField from " + + TABLE_ID_FASTQUERY_BQ_RESULTSET.getTable() + " order by TimestampField"; ConnectionSettings connectionSettings = - ConnectionSettings.newBuilder() - .setDefaultDataset(DatasetId.of(DATASET)) - .build(); + ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); Connection connection = bigquery.createConnection(connectionSettings); BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); ResultSet rs = bigQueryResultSet.getResultSet(); @@ -2164,23 +2155,6 @@ public void testExecuteQuerySinglePageTableRow() throws SQLException { assertEquals(3, bigQueryResultSet.getTotalRows()); -/* while (rs.next()){ - System.out.println(rs.getString("StringField")); - System.out.println(rs.getDouble("BigNumericField")); - System.out.println(rs.getBoolean("BooleanField")); - System.out.println(rs.getBytes("BytesField")); - System.out.println(rs.getInt("IntegerField")); - System.out.println(rs.getTimestamp("TimestampField")); - System.out.println(rs.getDate("DateField")); - System.out.println(rs.getDouble("FloatField")); - System.out.println(rs.getDouble("NumericField")); - System.out.println(rs.getTime("TimeField")); - System.out.println(rs.getString("DateTimeField")); - System.out.println(rs.getString("StructField")); - System.out.println(rs.getString("IntegerArrayField")); - System.out.println("\n\n>\n\n"); - }*/ - assertTrue(rs.next()); // first row // checking for the null or 0 column values assertNull(rs.getString("StringField")); @@ -2195,7 +2169,8 @@ public void testExecuteQuerySinglePageTableRow() throws SQLException { assertNull(rs.getTime("TimeField")); assertNull(rs.getString("DateTimeField")); assertNull(rs.getString("StructField")); - assertNull(rs.getString("IntegerArrayField")); + assertNull(rs.getString("StringArrayField")); + assertNull(rs.getString("GeographyField")); assertTrue(rs.next()); // second row // second row is non null, comparing the values @@ -2205,20 +2180,23 @@ public void testExecuteQuerySinglePageTableRow() throws SQLException { assertNotNull(rs.getBytes("BytesField")); assertEquals(1, rs.getInt("IntegerField")); /* -org.junit.ComparisonFailure: -Expected :2018-08-19 17:41:35.123456 -Actual :2018-08-19 17:41:35.22 - */ - // assertEquals("2018-08-19 17:41:35.123456", rs.getTimestamp("TimestampField").toString()); - assertEquals( java.sql.Date.valueOf("2018-08-19"), rs.getDate("DateField")); + TODO: Check the rounding issue + org.junit.ComparisonFailure: + Expected :2018-08-19 17:41:35.123456 + Actual :2018-08-19 17:41:35.22 + */ + // assertEquals("2018-08-19 17:41:35.123456", + // rs.getTimestamp("TimestampField").toString());//TODO: Check the rounding issue + assertEquals(java.sql.Date.valueOf("2018-08-19"), rs.getDate("DateField")); assertTrue(rs.getDouble("FloatField") == 10.1d); assertTrue(rs.getDouble("NumericField") == 100.0d); assertEquals(Time.valueOf(LocalTime.of(12, 11, 35, 123456)), rs.getTime("TimeField")); assertEquals("2018-08-19T12:11:35.123456", rs.getString("DateTimeField")); assertEquals("STRUCT('StructVal')", rs.getString("StructField")); - assertEquals("0", rs.getString("IntegerArrayField")); + assertEquals("one", rs.getString("StringArrayField")); + assertEquals("POINT(-122.35022 47.649154)", rs.getString("GeographyField")); - assertTrue(rs.next()); //third row + assertTrue(rs.next()); // third row assertFalse(rs.next()); // no 4th row in the table } From 424b35ca6608e1b01e2b9acd501d35f0e3d69bea Mon Sep 17 00:00:00 2001 From: stephwang Date: Mon, 11 Oct 2021 12:03:14 -0400 Subject: [PATCH 034/277] chore: manage checker-qual as a test dep We want to exclude this dep in our flattened dep tree --- pom.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pom.xml b/pom.xml index 3a6e003ac..46a6f740d 100644 --- a/pom.xml +++ b/pom.xml @@ -129,6 +129,13 @@ 1.5.7 test + + + org.checkerframework + checker-qual + 3.8.0 + test + org.assertj assertj-core From 50d1b2697ee9f0c561972ffe64eeebe1679382ab Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Mon, 11 Oct 2021 22:22:47 +0530 Subject: [PATCH 035/277] Fixed testExecuteQuerySinglePageTableRow testcase --- .../com/google/cloud/bigquery/it/ITBigQueryTest.java | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 53fc03474..df9eee627 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -2179,14 +2179,10 @@ public void testExecuteQuerySinglePageTableRow() throws SQLException { assertFalse(rs.getBoolean("BooleanField")); assertNotNull(rs.getBytes("BytesField")); assertEquals(1, rs.getInt("IntegerField")); - /* - TODO: Check the rounding issue - org.junit.ComparisonFailure: - Expected :2018-08-19 17:41:35.123456 - Actual :2018-08-19 17:41:35.22 - */ - // assertEquals("2018-08-19 17:41:35.123456", - // rs.getTimestamp("TimestampField").toString());//TODO: Check the rounding issue + /* assertEquals( + "2018-08-19 17:41:35.12", + rs.getTimestamp("TimestampField").toString()); // precision up to milliseconds*/ + // TODO assertEquals(java.sql.Date.valueOf("2018-08-19"), rs.getDate("DateField")); assertTrue(rs.getDouble("FloatField") == 10.1d); assertTrue(rs.getDouble("NumericField") == 100.0d); From e26f751a69f241ef231b69b42fd3e13c40f68e6a Mon Sep 17 00:00:00 2001 From: stephwang Date: Mon, 11 Oct 2021 13:35:09 -0400 Subject: [PATCH 036/277] fix lint issue --- .../test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 125f506ac..8306500e6 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -133,8 +133,8 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Time; -import java.time.LocalTime; import java.time.Instant; +import java.time.LocalTime; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; From 9f94cae18edbc712902b97f1ef8d000b5096e106 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 12 Oct 2021 08:55:16 +0530 Subject: [PATCH 037/277] Included a couple of RecordType attributes in testExecuteQuerySinglePageTableRow --- .../com/google/cloud/bigquery/it/ITBigQueryTest.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 06a91641b..5617689dc 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -411,6 +411,12 @@ public class ITBigQueryTest { .setMode(Field.Mode.NULLABLE) .build(), Field.newBuilder("GeographyField", LegacySQLTypeName.GEOGRAPHY) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("RecordField_BytesField", LegacySQLTypeName.BYTES) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("RecordField_BooleanField", LegacySQLTypeName.BOOLEAN) .setMode(Field.Mode.NULLABLE) .build()); @@ -2176,7 +2182,7 @@ public void testQueryExternalHivePartitioningOptionCustomLayout() throws Interru public void testExecuteQuerySinglePageTableRow() throws SQLException { String query = "select StringField, BigNumericField, BooleanField, BytesField, IntegerField, TimestampField, FloatField, " - + "NumericField, TimeField, DateField, DateTimeField, StructField, StringArrayField , GeographyField from " + + "NumericField, TimeField, DateField, DateTimeField, StructField, StringArrayField , GeographyField, RecordField.BytesField, RecordField.BooleanField from " + TABLE_ID_FASTQUERY_BQ_RESULTSET.getTable() + " order by TimestampField"; ConnectionSettings connectionSettings = @@ -2206,6 +2212,8 @@ public void testExecuteQuerySinglePageTableRow() throws SQLException { assertNull(rs.getString("StructField")); assertNull(rs.getString("StringArrayField")); assertNull(rs.getString("GeographyField")); + assertNull(rs.getBytes("RecordField_BytesField")); + assertFalse(rs.getBoolean("RecordField_BooleanField")); assertTrue(rs.next()); // second row // second row is non null, comparing the values @@ -2226,6 +2234,8 @@ public void testExecuteQuerySinglePageTableRow() throws SQLException { assertEquals("STRUCT('StructVal')", rs.getString("StructField")); assertEquals("one", rs.getString("StringArrayField")); assertEquals("POINT(-122.35022 47.649154)", rs.getString("GeographyField")); + assertNotNull(rs.getBytes("RecordField_BytesField")); + assertTrue(rs.getBoolean("RecordField_BooleanField")); assertTrue(rs.next()); // third row assertFalse(rs.next()); // no 4th row in the table From d6759bcb61133e8df54ff8594102ed28b1ebace8 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 12 Oct 2021 19:44:56 +0530 Subject: [PATCH 038/277] Switching to StandardSQLType for testExecuteQuerySinglePageTableRow --- .../cloud/bigquery/it/ITBigQueryTest.java | 184 +++++++++++------- 1 file changed, 114 insertions(+), 70 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 5617689dc..daa0ceaa6 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -168,7 +168,6 @@ public class ITBigQueryTest { private static final String OTHER_DATASET = RemoteBigQueryHelper.generateDatasetName(); private static final String MODEL_DATASET = RemoteBigQueryHelper.generateDatasetName(); private static final String ROUTINE_DATASET = RemoteBigQueryHelper.generateDatasetName(); - private static final String BQ_RESULTSET_DATASET = RemoteBigQueryHelper.generateDatasetName(); private static final String PROJECT_ID = ServiceOptions.getDefaultProjectId(); private static final String RANDOM_ID = UUID.randomUUID().toString().substring(0, 8); private static final String CLOUD_SAMPLES_DATA = @@ -182,26 +181,6 @@ public class ITBigQueryTest { .setMode(Field.Mode.NULLABLE) .setDescription("TimestampDescription") .build(); - private static final Field TIME_FIELD_SCHEMA = - Field.newBuilder("TimeField", LegacySQLTypeName.TIME) - .setMode(Field.Mode.NULLABLE) - .setDescription("TimeDescription") - .build(); - private static final Field DATE_FIELD_SCHEMA = - Field.newBuilder("DateField", LegacySQLTypeName.DATE) - .setMode(Field.Mode.NULLABLE) - .setDescription("DateDescription") - .build(); - private static final Field DATE_TIME_FIELD_SCHEMA = - Field.newBuilder("DateTimeField", LegacySQLTypeName.DATETIME) - .setMode(Field.Mode.NULLABLE) - .setDescription("DateTimeDescription") - .build(); - private static final Field STRUCT_FIELD_SCHEMA = - Field.newBuilder("StructField", LegacySQLTypeName.STRING) - .setMode(Field.Mode.NULLABLE) - .setDescription("StructDescription") - .build(); private static final Field STRING_FIELD_SCHEMA = Field.newBuilder("StringField", LegacySQLTypeName.STRING) .setMode(Field.Mode.NULLABLE) @@ -212,11 +191,6 @@ public class ITBigQueryTest { .setMode(Field.Mode.REPEATED) .setDescription("IntegerArrayDescription") .build(); - private static final Field STRING_ARRAY_FIELD_SCHEMA = - Field.newBuilder("StringArrayField", LegacySQLTypeName.STRING) - .setMode(Field.Mode.REPEATED) - .setDescription("StringArrayDescription") - .build(); private static final Field BOOLEAN_FIELD_SCHEMA = Field.newBuilder("BooleanField", LegacySQLTypeName.BOOLEAN) .setMode(Field.Mode.NULLABLE) @@ -304,26 +278,104 @@ public class ITBigQueryTest { private static final Schema BQ_RESULTSET_SCHEMA = Schema.of( - TIMESTAMP_FIELD_SCHEMA, - STRING_FIELD_SCHEMA, - INTEGER_ARRAY_FIELD_SCHEMA, - BOOLEAN_FIELD_SCHEMA, - BYTES_FIELD_SCHEMA, - RECORD_FIELD_SCHEMA, - INTEGER_FIELD_SCHEMA, - FLOAT_FIELD_SCHEMA, - GEOGRAPHY_FIELD_SCHEMA, - NUMERIC_FIELD_SCHEMA, - BIGNUMERIC_FIELD_SCHEMA, - BIGNUMERIC_FIELD_SCHEMA1, - BIGNUMERIC_FIELD_SCHEMA2, - BIGNUMERIC_FIELD_SCHEMA3, - BIGNUMERIC_FIELD_SCHEMA4, - TIME_FIELD_SCHEMA, - DATE_FIELD_SCHEMA, - DATE_TIME_FIELD_SCHEMA, - STRUCT_FIELD_SCHEMA, - STRING_ARRAY_FIELD_SCHEMA); + Field.newBuilder("TimestampField", StandardSQLTypeName.TIMESTAMP) + .setMode(Field.Mode.NULLABLE) + .setDescription("TimestampDescription") + .build(), + Field.newBuilder("StringField", StandardSQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .setDescription("StringDescription") + .build(), + Field.newBuilder("IntegerArrayField", StandardSQLTypeName.NUMERIC) + .setMode(Field.Mode.REPEATED) + .setDescription("IntegerArrayDescription") + .build(), + Field.newBuilder("BooleanField", StandardSQLTypeName.BOOL) + .setMode(Field.Mode.NULLABLE) + .setDescription("BooleanDescription") + .build(), + Field.newBuilder("BytesField", StandardSQLTypeName.BYTES) + .setMode(Field.Mode.NULLABLE) + .setDescription("BytesDescription") + .build(), + Field.newBuilder( + "RecordField", + StandardSQLTypeName.STRUCT, + Field.newBuilder("TimestampField", StandardSQLTypeName.TIMESTAMP) + .setMode(Field.Mode.NULLABLE) + .setDescription("TimestampDescription") + .build(), + Field.newBuilder("StringField", StandardSQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .setDescription("StringDescription") + .build(), + Field.newBuilder("IntegerArrayField", StandardSQLTypeName.NUMERIC) + .setMode(Field.Mode.REPEATED) + .setDescription("IntegerArrayDescription") + .build(), + Field.newBuilder("BooleanField", StandardSQLTypeName.BOOL) + .setMode(Field.Mode.NULLABLE) + .setDescription("BooleanDescription") + .build(), + Field.newBuilder("BytesField", StandardSQLTypeName.BYTES) + .setMode(Field.Mode.NULLABLE) + .setDescription("BytesDescription") + .build()) + .setMode(Field.Mode.REQUIRED) + .setDescription("RecordDescription") + .build(), + Field.newBuilder("IntegerField", StandardSQLTypeName.NUMERIC) + .setMode(Field.Mode.NULLABLE) + .setDescription("IntegerDescription") + .build(), + Field.newBuilder("FloatField", StandardSQLTypeName.NUMERIC) + .setMode(Field.Mode.NULLABLE) + .setDescription("FloatDescription") + .build(), + Field.newBuilder("GeographyField", StandardSQLTypeName.GEOGRAPHY) + .setMode(Field.Mode.NULLABLE) + .setDescription("GeographyDescription") + .build(), + Field.newBuilder("NumericField", StandardSQLTypeName.NUMERIC) + .setMode(Field.Mode.NULLABLE) + .setDescription("NumericDescription") + .build(), + Field.newBuilder("BigNumericField", StandardSQLTypeName.BIGNUMERIC) + .setMode(Field.Mode.NULLABLE) + .setDescription("BigNumericDescription") + .build(), + Field.newBuilder("BigNumericField1", StandardSQLTypeName.BIGNUMERIC) + .setMode(Field.Mode.NULLABLE) + .setDescription("BigNumeric1Description") + .build(), + Field.newBuilder("BigNumericField2", StandardSQLTypeName.BIGNUMERIC) + .setMode(Field.Mode.NULLABLE) + .setDescription("BigNumeric2Description") + .build(), + Field.newBuilder("BigNumericField3", StandardSQLTypeName.BIGNUMERIC) + .setMode(Field.Mode.NULLABLE) + .setDescription("BigNumeric3Description") + .build(), + Field.newBuilder("BigNumericField4", StandardSQLTypeName.BIGNUMERIC) + .setMode(Field.Mode.NULLABLE) + .setDescription("BigNumeric4Description") + .build(), + Field.newBuilder("TimeField", StandardSQLTypeName.TIME) + .setMode(Field.Mode.NULLABLE) + .setDescription("TimeDescription") + .build(), + Field.newBuilder("DateField", StandardSQLTypeName.DATE) + .setMode(Field.Mode.NULLABLE) + .setDescription("DateDescription") + .build(), + Field.newBuilder("DateTimeField", StandardSQLTypeName.DATETIME) + .setMode(Field.Mode.NULLABLE) + .setDescription("DateTimeDescription") + .build(), + Field.newBuilder("StringArrayField", StandardSQLTypeName.STRING) + .setMode(Field.Mode.REPEATED) + .setDescription("StringArrayDescription") + .build()); private static final Field DDL_TIMESTAMP_FIELD_SCHEMA = Field.newBuilder("TimestampField", LegacySQLTypeName.TIMESTAMP) @@ -371,52 +423,49 @@ public class ITBigQueryTest { private static final Schema BQ_RESULTSET_EXPECTED_SCHEMA = Schema.of( - Field.newBuilder("StringField", LegacySQLTypeName.STRING) - .setMode(Field.Mode.NULLABLE) - .build(), - Field.newBuilder("BigNumericField", LegacySQLTypeName.BIGNUMERIC) + Field.newBuilder("StringField", StandardSQLTypeName.STRING) .setMode(Field.Mode.NULLABLE) .build(), - Field.newBuilder("BooleanField", LegacySQLTypeName.BOOLEAN) + Field.newBuilder("BigNumericField", StandardSQLTypeName.BIGNUMERIC) .setMode(Field.Mode.NULLABLE) .build(), - Field.newBuilder("BytesField", LegacySQLTypeName.BYTES) + Field.newBuilder("BooleanField", StandardSQLTypeName.BOOL) .setMode(Field.Mode.NULLABLE) .build(), - Field.newBuilder("IntegerField", LegacySQLTypeName.INTEGER) + Field.newBuilder("BytesField", StandardSQLTypeName.BYTES) .setMode(Field.Mode.NULLABLE) .build(), - Field.newBuilder("TimestampField", LegacySQLTypeName.TIMESTAMP) + Field.newBuilder("IntegerField", StandardSQLTypeName.NUMERIC) .setMode(Field.Mode.NULLABLE) .build(), - Field.newBuilder("FloatField", LegacySQLTypeName.FLOAT) + Field.newBuilder("TimestampField", StandardSQLTypeName.TIMESTAMP) .setMode(Field.Mode.NULLABLE) .build(), - Field.newBuilder("NumericField", LegacySQLTypeName.NUMERIC) + Field.newBuilder("FloatField", StandardSQLTypeName.NUMERIC) .setMode(Field.Mode.NULLABLE) .build(), - Field.newBuilder("TimeField", LegacySQLTypeName.TIME) + Field.newBuilder("NumericField", StandardSQLTypeName.NUMERIC) .setMode(Field.Mode.NULLABLE) .build(), - Field.newBuilder("DateField", LegacySQLTypeName.DATE) + Field.newBuilder("TimeField", StandardSQLTypeName.TIME) .setMode(Field.Mode.NULLABLE) .build(), - Field.newBuilder("DateTimeField", LegacySQLTypeName.DATETIME) + Field.newBuilder("DateField", StandardSQLTypeName.DATE) .setMode(Field.Mode.NULLABLE) .build(), - Field.newBuilder("StructField", LegacySQLTypeName.STRING) + Field.newBuilder("DateTimeField", StandardSQLTypeName.DATETIME) .setMode(Field.Mode.NULLABLE) .build(), - Field.newBuilder("StringArrayField", LegacySQLTypeName.STRING) + Field.newBuilder("StringArrayField", StandardSQLTypeName.STRING) .setMode(Field.Mode.NULLABLE) .build(), - Field.newBuilder("GeographyField", LegacySQLTypeName.GEOGRAPHY) + Field.newBuilder("GeographyField", StandardSQLTypeName.GEOGRAPHY) .setMode(Field.Mode.NULLABLE) .build(), - Field.newBuilder("RecordField_BytesField", LegacySQLTypeName.BYTES) + Field.newBuilder("RecordField_BytesField", StandardSQLTypeName.BYTES) .setMode(Field.Mode.NULLABLE) .build(), - Field.newBuilder("RecordField_BooleanField", LegacySQLTypeName.BOOLEAN) + Field.newBuilder("RecordField_BooleanField", StandardSQLTypeName.BOOL) .setMode(Field.Mode.NULLABLE) .build()); @@ -559,7 +608,6 @@ public class ITBigQueryTest { + " \"TimeField\": null," + " \"DateField\": null," + " \"DateTimeField\": null," - + " \"StructField\": null," + " \"StringArrayField\": null" + "}\n" + "{" @@ -591,7 +639,6 @@ public class ITBigQueryTest { + " \"TimeField\": \"12:11:35.123456\"," + " \"DateField\": \"2018-08-19\"," + " \"DateTimeField\": \"2018-08-19 12:11:35.123456\"," - + " \"StructField\": \"STRUCT('StructVal')\"," + " \"StringArrayField\": [\"one\", \"two\"]" + "}"; private static final String JSON_CONTENT_SIMPLE = @@ -2182,7 +2229,7 @@ public void testQueryExternalHivePartitioningOptionCustomLayout() throws Interru public void testExecuteQuerySinglePageTableRow() throws SQLException { String query = "select StringField, BigNumericField, BooleanField, BytesField, IntegerField, TimestampField, FloatField, " - + "NumericField, TimeField, DateField, DateTimeField, StructField, StringArrayField , GeographyField, RecordField.BytesField, RecordField.BooleanField from " + + "NumericField, TimeField, DateField, DateTimeField, StringArrayField , GeographyField, RecordField.BytesField, RecordField.BooleanField from " + TABLE_ID_FASTQUERY_BQ_RESULTSET.getTable() + " order by TimestampField"; ConnectionSettings connectionSettings = @@ -2193,7 +2240,6 @@ public void testExecuteQuerySinglePageTableRow() throws SQLException { Schema sc = bigQueryResultSet.getSchema(); assertEquals(BQ_RESULTSET_EXPECTED_SCHEMA, sc); // match the schema - assertEquals(3, bigQueryResultSet.getTotalRows()); assertTrue(rs.next()); // first row @@ -2209,7 +2255,6 @@ public void testExecuteQuerySinglePageTableRow() throws SQLException { assertTrue(rs.getDouble("NumericField") == 0.0d); assertNull(rs.getTime("TimeField")); assertNull(rs.getString("DateTimeField")); - assertNull(rs.getString("StructField")); assertNull(rs.getString("StringArrayField")); assertNull(rs.getString("GeographyField")); assertNull(rs.getBytes("RecordField_BytesField")); @@ -2231,7 +2276,6 @@ public void testExecuteQuerySinglePageTableRow() throws SQLException { assertTrue(rs.getDouble("NumericField") == 100.0d); assertEquals(Time.valueOf(LocalTime.of(12, 11, 35, 123456)), rs.getTime("TimeField")); assertEquals("2018-08-19T12:11:35.123456", rs.getString("DateTimeField")); - assertEquals("STRUCT('StructVal')", rs.getString("StructField")); assertEquals("one", rs.getString("StringArrayField")); assertEquals("POINT(-122.35022 47.649154)", rs.getString("GeographyField")); assertNotNull(rs.getBytes("RecordField_BytesField")); From c93e08cbefb143e98d0500f551c3f9959d28e097 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 12 Oct 2021 20:15:43 +0530 Subject: [PATCH 039/277] Fixed getTimestamp assertion --- .../java/com/google/cloud/bigquery/it/ITBigQueryTest.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index daa0ceaa6..6ba140b07 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -611,7 +611,7 @@ public class ITBigQueryTest { + " \"StringArrayField\": null" + "}\n" + "{" - + " \"TimestampField\": \"2018-08-19 12:11:35.220 UTC\"," + + " \"TimestampField\": \"2018-08-19 12:11:35.123456 UTC\"," + " \"StringField\": \"StringValue1\"," + " \"IntegerArrayField\": [\"0\", \"1\"]," + " \"BooleanField\": \"false\"," @@ -2267,10 +2267,7 @@ public void testExecuteQuerySinglePageTableRow() throws SQLException { assertFalse(rs.getBoolean("BooleanField")); assertNotNull(rs.getBytes("BytesField")); assertEquals(1, rs.getInt("IntegerField")); - /* assertEquals( - "2018-08-19 17:41:35.12", - rs.getTimestamp("TimestampField").toString()); // precision up to milliseconds*/ - // TODO + assertEquals("2018-08-19 17:41:35.123", rs.getTimestamp("TimestampField").toString()); assertEquals(java.sql.Date.valueOf("2018-08-19"), rs.getDate("DateField")); assertTrue(rs.getDouble("FloatField") == 10.1d); assertTrue(rs.getDouble("NumericField") == 100.0d); From cf7c941dcd239979b606a7becad196b11389f4f9 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 12 Oct 2021 20:23:14 +0530 Subject: [PATCH 040/277] Added assertion for the third row - testExecuteQuerySinglePageTableRow --- .../cloud/bigquery/it/ITBigQueryTest.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 6ba140b07..a9c388205 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -2279,6 +2279,23 @@ public void testExecuteQuerySinglePageTableRow() throws SQLException { assertTrue(rs.getBoolean("RecordField_BooleanField")); assertTrue(rs.next()); // third row + // 3rd row + assertEquals("StringValue1", rs.getString("StringField")); + assertTrue(rs.getDouble("BigNumericField") == 0.3333333333333333d); + assertFalse(rs.getBoolean("BooleanField")); + assertNotNull(rs.getBytes("BytesField")); + assertEquals(1, rs.getInt("IntegerField")); + assertEquals("2018-08-19 17:41:35.123", rs.getTimestamp("TimestampField").toString()); + assertEquals(java.sql.Date.valueOf("2018-08-19"), rs.getDate("DateField")); + assertTrue(rs.getDouble("FloatField") == 10.1d); + assertTrue(rs.getDouble("NumericField") == 100.0d); + assertEquals(Time.valueOf(LocalTime.of(12, 11, 35, 123456)), rs.getTime("TimeField")); + assertEquals("2018-08-19T12:11:35.123456", rs.getString("DateTimeField")); + assertEquals("two", rs.getString("StringArrayField")); + assertEquals("POINT(-122.35022 47.649154)", rs.getString("GeographyField")); + assertNotNull(rs.getBytes("RecordField_BytesField")); + assertTrue(rs.getBoolean("RecordField_BooleanField")); + assertFalse(rs.next()); // no 4th row in the table } From 68d239eb2696676b075daf56cb2834656bebcf04 Mon Sep 17 00:00:00 2001 From: stephwang Date: Tue, 12 Oct 2021 16:17:18 -0400 Subject: [PATCH 041/277] add testcase for query struct --- .../google/cloud/bigquery/ConnectionImpl.java | 4 ++++ .../cloud/bigquery/it/ITBigQueryTest.java | 22 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index e5bd8eb8c..7e000d385 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -390,6 +390,8 @@ private QueryRequest createQueryRequest( } content.setQuery(sql); content.setRequestId(requestId); + // The new Connection interface only supports StandardSQL dialect + content.setUseLegacySql(false); return content; } @@ -486,6 +488,8 @@ private com.google.api.services.bigquery.model.Job createQueryJob( if (labels != null) { configurationPb.setLabels(labels); } + // The new Connection interface only supports StandardSQL dialect + queryConfigurationPb.setUseLegacySql(false); configurationPb.setQuery(queryConfigurationPb); com.google.api.services.bigquery.model.Job jobPb = diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index a9c388205..93fb01a9d 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -2299,6 +2299,28 @@ public void testExecuteQuerySinglePageTableRow() throws SQLException { assertFalse(rs.next()); // no 4th row in the table } + @Test + public void testExecuteSelectStruct() throws SQLException { + String query = "select (STRUCT(\"Vancouver\" as city, 5 as years)) as address"; + ConnectionSettings connectionSettings = + ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); + Connection connection = bigquery.createConnection(connectionSettings); + BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); + assertEquals(1, bigQueryResultSet.getTotalRows()); + + Schema schema = bigQueryResultSet.getSchema(); + assertEquals(schema.getFields().get(0).getName(), "address"); + assertEquals(schema.getFields().get(0).getMode(), Field.Mode.NULLABLE); + // Backend is currently returning RECORD which is LegacySQLTypeName. + assertEquals(schema.getFields().get(0).getType(), LegacySQLTypeName.RECORD); + + ResultSet rs = bigQueryResultSet.getResultSet(); + assertTrue(rs.next()); + // TODO: Figure out what this is supposed to return + // Object o = rs.getObject("address"); // is this supposed to return the reinterpreted JSON? + assertFalse(rs.next()); // only 1 row of data + } + @Test public void testFastQueryMultipleRuns() throws InterruptedException { String query = From 0a5b224cb324e6618acb2c540ca5643e1dd900b2 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 14 Oct 2021 20:57:29 +0530 Subject: [PATCH 042/277] Fixed getObject(String fieldName) --- .../cloud/bigquery/BigQueryResultSetImpl.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index 978e025ba..a19c9267b 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -16,7 +16,6 @@ package com.google.cloud.bigquery; -import com.google.api.services.bigquery.model.TableRow; import java.math.BigDecimal; import java.sql.Date; import java.sql.ResultSet; @@ -82,15 +81,16 @@ private boolean isEndOfStream(T cursor) { @Override public Object getObject(String fieldName) throws SQLException { - if (cursor != null && cursor instanceof TableRow) { - TableRow currentRow = (TableRow) cursor; - if (fieldName == null) { - throw new SQLException("fieldName can't be null"); - } - return currentRow.get(fieldName); + if (fieldName == null) { + throw new SQLException("fieldName can't be null"); } - // TODO: Add similar clauses for Apache Arrow - return null; + if (cursor == null) { + return null; + } else if (cursor instanceof FieldValueList) { + FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); + return fieldValue.getValue() == null ? null : fieldValue.getValue(); + } + return null; // TODO: Implementation for Arrow } @Override From 5094c4da9a3b84463412695bbb3412debcf704e7 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 14 Oct 2021 20:58:39 +0530 Subject: [PATCH 043/277] Updated testExecuteQuerySinglePageTableRow to test nested Arrays --- .../cloud/bigquery/it/ITBigQueryTest.java | 64 +++++++------------ 1 file changed, 22 insertions(+), 42 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 93fb01a9d..730fe51ef 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -371,10 +371,6 @@ public class ITBigQueryTest { Field.newBuilder("DateTimeField", StandardSQLTypeName.DATETIME) .setMode(Field.Mode.NULLABLE) .setDescription("DateTimeDescription") - .build(), - Field.newBuilder("StringArrayField", StandardSQLTypeName.STRING) - .setMode(Field.Mode.REPEATED) - .setDescription("StringArrayDescription") .build()); private static final Field DDL_TIMESTAMP_FIELD_SCHEMA = @@ -456,17 +452,17 @@ public class ITBigQueryTest { Field.newBuilder("DateTimeField", StandardSQLTypeName.DATETIME) .setMode(Field.Mode.NULLABLE) .build(), - Field.newBuilder("StringArrayField", StandardSQLTypeName.STRING) - .setMode(Field.Mode.NULLABLE) - .build(), Field.newBuilder("GeographyField", StandardSQLTypeName.GEOGRAPHY) .setMode(Field.Mode.NULLABLE) .build(), - Field.newBuilder("RecordField_BytesField", StandardSQLTypeName.BYTES) + Field.newBuilder("BytesField_1", StandardSQLTypeName.BYTES) .setMode(Field.Mode.NULLABLE) .build(), - Field.newBuilder("RecordField_BooleanField", StandardSQLTypeName.BOOL) + Field.newBuilder("BooleanField_1", StandardSQLTypeName.BOOL) .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("IntegerArrayField", StandardSQLTypeName.NUMERIC) + .setMode(Field.Mode.REPEATED) .build()); private static final Schema QUERY_RESULT_SCHEMA_BIGNUMERIC = @@ -607,13 +603,12 @@ public class ITBigQueryTest { + " \"BigNumericField4\": null," + " \"TimeField\": null," + " \"DateField\": null," - + " \"DateTimeField\": null," - + " \"StringArrayField\": null" + + " \"DateTimeField\": null" + "}\n" + "{" + " \"TimestampField\": \"2018-08-19 12:11:35.123456 UTC\"," + " \"StringField\": \"StringValue1\"," - + " \"IntegerArrayField\": [\"0\", \"1\"]," + + " \"IntegerArrayField\": [1,2,3,4]," + " \"BooleanField\": \"false\"," + " \"BytesField\": \"" + BYTES_BASE64 @@ -621,7 +616,7 @@ public class ITBigQueryTest { + " \"RecordField\": {" + " \"TimestampField\": \"1969-07-20 20:18:04 UTC\"," + " \"StringField\": null," - + " \"IntegerArrayField\": [\"1\",\"0\"]," + + " \"IntegerArrayField\": [1,0]," + " \"BooleanField\": \"true\"," + " \"BytesField\": \"" + BYTES_BASE64 @@ -638,8 +633,7 @@ public class ITBigQueryTest { + " \"BigNumericField4\": \"-578960446186580977117854925043439539266.34992332820282019728792003956564819968\"," + " \"TimeField\": \"12:11:35.123456\"," + " \"DateField\": \"2018-08-19\"," - + " \"DateTimeField\": \"2018-08-19 12:11:35.123456\"," - + " \"StringArrayField\": [\"one\", \"two\"]" + + " \"DateTimeField\": \"2018-08-19 12:11:35.123456\"" + "}"; private static final String JSON_CONTENT_SIMPLE = "{" @@ -2229,7 +2223,7 @@ public void testQueryExternalHivePartitioningOptionCustomLayout() throws Interru public void testExecuteQuerySinglePageTableRow() throws SQLException { String query = "select StringField, BigNumericField, BooleanField, BytesField, IntegerField, TimestampField, FloatField, " - + "NumericField, TimeField, DateField, DateTimeField, StringArrayField , GeographyField, RecordField.BytesField, RecordField.BooleanField from " + + "NumericField, TimeField, DateField, DateTimeField , GeographyField, RecordField.BytesField, RecordField.BooleanField, IntegerArrayField from " + TABLE_ID_FASTQUERY_BQ_RESULTSET.getTable() + " order by TimestampField"; ConnectionSettings connectionSettings = @@ -2240,7 +2234,7 @@ public void testExecuteQuerySinglePageTableRow() throws SQLException { Schema sc = bigQueryResultSet.getSchema(); assertEquals(BQ_RESULTSET_EXPECTED_SCHEMA, sc); // match the schema - assertEquals(3, bigQueryResultSet.getTotalRows()); + assertEquals(2, bigQueryResultSet.getTotalRows()); // Expecting 2 rows assertTrue(rs.next()); // first row // checking for the null or 0 column values @@ -2255,10 +2249,9 @@ public void testExecuteQuerySinglePageTableRow() throws SQLException { assertTrue(rs.getDouble("NumericField") == 0.0d); assertNull(rs.getTime("TimeField")); assertNull(rs.getString("DateTimeField")); - assertNull(rs.getString("StringArrayField")); assertNull(rs.getString("GeographyField")); - assertNull(rs.getBytes("RecordField_BytesField")); - assertFalse(rs.getBoolean("RecordField_BooleanField")); + assertNull(rs.getBytes("BytesField_1")); + assertFalse(rs.getBoolean("BooleanField_1")); assertTrue(rs.next()); // second row // second row is non null, comparing the values @@ -2273,30 +2266,17 @@ public void testExecuteQuerySinglePageTableRow() throws SQLException { assertTrue(rs.getDouble("NumericField") == 100.0d); assertEquals(Time.valueOf(LocalTime.of(12, 11, 35, 123456)), rs.getTime("TimeField")); assertEquals("2018-08-19T12:11:35.123456", rs.getString("DateTimeField")); - assertEquals("one", rs.getString("StringArrayField")); assertEquals("POINT(-122.35022 47.649154)", rs.getString("GeographyField")); - assertNotNull(rs.getBytes("RecordField_BytesField")); - assertTrue(rs.getBoolean("RecordField_BooleanField")); - - assertTrue(rs.next()); // third row - // 3rd row - assertEquals("StringValue1", rs.getString("StringField")); - assertTrue(rs.getDouble("BigNumericField") == 0.3333333333333333d); - assertFalse(rs.getBoolean("BooleanField")); - assertNotNull(rs.getBytes("BytesField")); - assertEquals(1, rs.getInt("IntegerField")); - assertEquals("2018-08-19 17:41:35.123", rs.getTimestamp("TimestampField").toString()); - assertEquals(java.sql.Date.valueOf("2018-08-19"), rs.getDate("DateField")); - assertTrue(rs.getDouble("FloatField") == 10.1d); - assertTrue(rs.getDouble("NumericField") == 100.0d); - assertEquals(Time.valueOf(LocalTime.of(12, 11, 35, 123456)), rs.getTime("TimeField")); - assertEquals("2018-08-19T12:11:35.123456", rs.getString("DateTimeField")); - assertEquals("two", rs.getString("StringArrayField")); - assertEquals("POINT(-122.35022 47.649154)", rs.getString("GeographyField")); - assertNotNull(rs.getBytes("RecordField_BytesField")); - assertTrue(rs.getBoolean("RecordField_BooleanField")); + assertNotNull(rs.getBytes("BytesField_1")); + assertTrue(rs.getBoolean("BooleanField_1")); + assertTrue( + rs.getObject("IntegerArrayField") instanceof com.google.cloud.bigquery.FieldValueList); + FieldValueList integerArrayFieldValue = + (com.google.cloud.bigquery.FieldValueList) rs.getObject("IntegerArrayField"); + assertEquals(4, integerArrayFieldValue.size()); // Array has 4 elements + assertEquals(3, (integerArrayFieldValue.get(2).getNumericValue()).intValue()); - assertFalse(rs.next()); // no 4th row in the table + assertFalse(rs.next()); // no 3rd row in the table } @Test From e53e9f7dcc231a2b9d9ffd7f83c40d72bb2ae0d5 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 14 Oct 2021 21:42:49 +0530 Subject: [PATCH 044/277] fixed getTimestamp assertion --- .../test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 730fe51ef..edcfa77f8 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -2260,7 +2260,7 @@ public void testExecuteQuerySinglePageTableRow() throws SQLException { assertFalse(rs.getBoolean("BooleanField")); assertNotNull(rs.getBytes("BytesField")); assertEquals(1, rs.getInt("IntegerField")); - assertEquals("2018-08-19 17:41:35.123", rs.getTimestamp("TimestampField").toString()); + assertEquals(1534680695123L, rs.getTimestamp("TimestampField").getTime()); assertEquals(java.sql.Date.valueOf("2018-08-19"), rs.getDate("DateField")); assertTrue(rs.getDouble("FloatField") == 10.1d); assertTrue(rs.getDouble("NumericField") == 100.0d); From 5f161eecd22ef7b47a2df75c22d5461de634029a Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Mon, 18 Oct 2021 11:42:50 +0530 Subject: [PATCH 045/277] Removing the columnIndex overloads from the AbstractClass --- .../cloud/bigquery/AbstractJdbcResultSet.java | 65 ------------------- 1 file changed, 65 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/AbstractJdbcResultSet.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/AbstractJdbcResultSet.java index 415350fe4..5b8246925 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/AbstractJdbcResultSet.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/AbstractJdbcResultSet.java @@ -44,18 +44,6 @@ public boolean wasNull() throws SQLException { throw new RuntimeException("Not implemented"); } - @Override - public String getString(int columnIndex) throws SQLException { - // TODO: Implement the logic - throw new RuntimeException("Not implemented"); - } - - @Override - public boolean getBoolean(int columnIndex) throws SQLException { - // TODO: Implement the logic - throw new RuntimeException("Not implemented"); - } - @Override public byte getByte(int columnIndex) throws SQLException { // TODO: Implement the logic @@ -74,60 +62,18 @@ public short getShort(int columnIndex) throws SQLException { throw new RuntimeException("Not implemented"); } - @Override - public int getInt(int columnIndex) throws SQLException { - // TODO: Implement the logic - throw new RuntimeException("Not implemented"); - } - - @Override - public long getLong(int columnIndex) throws SQLException { - // TODO: Implement the logic - throw new RuntimeException("Not implemented"); - } - @Override public float getFloat(int columnIndex) throws SQLException { // TODO: Implement the logic throw new RuntimeException("Not implemented"); } - @Override - public double getDouble(int columnIndex) throws SQLException { - // TODO: Implement the logic - throw new RuntimeException("Not implemented"); - } - @Override public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException { // TODO: Implement the logic throw new RuntimeException("Not implemented"); } - @Override - public byte[] getBytes(int columnIndex) throws SQLException { - // TODO: Implement the logic - throw new RuntimeException("Not implemented"); - } - - @Override - public Date getDate(int columnIndex) throws SQLException { - // TODO: Implement the logic - throw new RuntimeException("Not implemented"); - } - - @Override - public Time getTime(int columnIndex) throws SQLException { - // TODO: Implement the logic - throw new RuntimeException("Not implemented"); - } - - @Override - public Timestamp getTimestamp(int columnIndex) throws SQLException { - // TODO: Implement the logic - throw new RuntimeException("Not implemented"); - } - @Override public InputStream getAsciiStream(int columnIndex) throws SQLException { // TODO: Implement the logic @@ -164,12 +110,6 @@ public Reader getCharacterStream(String columnLabel) throws SQLException { throw new RuntimeException("Not implemented"); } - @Override - public BigDecimal getBigDecimal(int columnIndex) throws SQLException { - // TODO: Implement the logic - throw new RuntimeException("Not implemented"); - } - @Override public SQLWarning getWarnings() throws SQLException { return null; @@ -958,11 +898,6 @@ public ResultSetMetaData getMetaData() throws SQLException { throw new SQLFeatureNotSupportedException(); } - @Override - public Object getObject(int columnIndex) throws SQLException { - throw new SQLFeatureNotSupportedException(); - } - @Override public T unwrap(Class iface) throws SQLException { throw new SQLFeatureNotSupportedException(); From 68dea7a1d550bf7811aa8280610c82ab168a836e Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Mon, 18 Oct 2021 11:54:11 +0530 Subject: [PATCH 046/277] Implemented columnIndex overloads --- .../cloud/bigquery/BigQueryResultSetImpl.java | 178 +++++++++++++++--- 1 file changed, 157 insertions(+), 21 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index a19c9267b..1648ddf35 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -93,6 +93,17 @@ public Object getObject(String fieldName) throws SQLException { return null; // TODO: Implementation for Arrow } + @Override + public Object getObject(int columnIndex) throws SQLException { + if (cursor == null) { + return null; + } else if (cursor instanceof FieldValueList) { + FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); + return fieldValue.getValue() == null ? null : fieldValue.getValue(); + } + return null; // TODO: Implementation for Arrow + } + @Override public String getString(String fieldName) throws SQLException { if (fieldName == null) { @@ -107,6 +118,17 @@ public String getString(String fieldName) throws SQLException { return null; // TODO: Implementation for Arrow } + @Override + public String getString(int columnIndex) throws SQLException { + if (cursor == null) { + return null; + } else if (cursor instanceof FieldValueList) { + FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); + return fieldValue.getValue() == null ? null : fieldValue.getStringValue(); + } + return null; // TODO: Implementation for Arrow + } + @Override public int getInt(String fieldName) throws SQLException { if (fieldName == null) { @@ -122,6 +144,18 @@ public int getInt(String fieldName) throws SQLException { return 0; // TODO: Implementation for Arrow } + @Override + public int getInt(int columnIndex) throws SQLException { + if (cursor == null) { + return 0; // the column value; if the value is SQL NULL, the value returned is 0 as per + // java.sql.ResultSet definition + } else if (cursor instanceof FieldValueList) { + FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); + return fieldValue.getValue() == null ? 0 : fieldValue.getNumericValue().intValue(); + } + return 0; // TODO: Implementation for Arrow + } + @Override public long getLong(String fieldName) throws SQLException { if (fieldName == null) { @@ -137,6 +171,18 @@ public long getLong(String fieldName) throws SQLException { return 0L; // TODO: Implementation for Arrow } + @Override + public long getLong(int columnIndex) throws SQLException { + if (cursor == null) { + return 0L; // the column value; if the value is SQL NULL, the value returned is 0 as per + // java.sql.ResultSet definition + } else if (cursor instanceof FieldValueList) { + FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); + return fieldValue.getValue() == null ? 0L : fieldValue.getNumericValue().longValue(); + } + return 0L; // TODO: Implementation for Arrow + } + @Override public double getDouble(String fieldName) throws SQLException { if (fieldName == null) { @@ -152,6 +198,18 @@ public double getDouble(String fieldName) throws SQLException { return 0d; // TODO: Implementation for Arrow } + @Override + public double getDouble(int columnIndex) throws SQLException { + if (cursor == null) { + return 0d; // the column value; if the value is SQL NULL, the value returned is 0 as per + // java.sql.ResultSet definition + } else if (cursor instanceof FieldValueList) { + FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); + return fieldValue.getValue() == null ? 0d : fieldValue.getNumericValue().doubleValue(); + } + return 0d; // TODO: Implementation for Arrow + } + @Override public BigDecimal getBigDecimal(String fieldName) throws SQLException { if (fieldName == null) { @@ -170,6 +228,21 @@ public BigDecimal getBigDecimal(String fieldName) throws SQLException { return null; // TODO: Implementation for Arrow } + @Override + public BigDecimal getBigDecimal(int columnIndex) throws SQLException { + if (cursor == null) { + return null; // the column value (full precision); if the value is SQL NULL, the value + // returned is null in the Java programming language. as per + // java.sql.ResultSet definition + } else if (cursor instanceof FieldValueList) { + FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); + return fieldValue.getValue() == null + ? null + : BigDecimal.valueOf(fieldValue.getNumericValue().doubleValue()); + } + return null; // TODO: Implementation for Arrow + } + @Override public boolean getBoolean(String fieldName) throws SQLException { if (fieldName == null) { @@ -184,6 +257,17 @@ public boolean getBoolean(String fieldName) throws SQLException { return false; // TODO: Implementation for Arrow } + @Override + public boolean getBoolean(int columnIndex) throws SQLException { + if (cursor == null) { + return false; // if the value is SQL NULL, the value returned is false + } else if (cursor instanceof FieldValueList) { + FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); + return fieldValue.getValue() != null && fieldValue.getBooleanValue(); + } + return false; // TODO: Implementation for Arrow + } + @Override public byte[] getBytes(String fieldName) throws SQLException { if (fieldName == null) { @@ -198,6 +282,17 @@ public byte[] getBytes(String fieldName) throws SQLException { return null; // TODO: Implementation for Arrow } + @Override + public byte[] getBytes(int columnIndex) throws SQLException { + if (cursor == null) { + return null; // if the value is SQL NULL, the value returned is null + } else if (cursor instanceof FieldValueList) { + FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); + return fieldValue.getValue() == null ? null : fieldValue.getBytesValue(); + } + return null; // TODO: Implementation for Arrow + } + @Override public Timestamp getTimestamp(String fieldName) throws SQLException { if (fieldName == null) { @@ -217,6 +312,22 @@ public Timestamp getTimestamp(String fieldName) throws SQLException { return null; // TODO: Implementation for Arrow } + @Override + public Timestamp getTimestamp(int columnIndex) throws SQLException { + if (cursor == null) { + return null; // if the value is SQL NULL, the value returned is null + } else if (cursor instanceof FieldValueList) { + FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); + return fieldValue.getValue() == null + ? null + : new Timestamp( + fieldValue.getTimestampValue() + / 1000); // getTimestampValue returns time in microseconds, and TimeStamp + // expects it in millis + } + return null; // TODO: Implementation for Arrow + } + @Override public Time getTime(String fieldName) throws SQLException { if (fieldName == null) { @@ -226,30 +337,44 @@ public Time getTime(String fieldName) throws SQLException { return null; // if the value is SQL NULL, the value returned is null } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); - if (fieldValue.getValue() != null) { - // Time ranges from 00:00:00 to 23:59:59.99999. in BigQuery. Parsing it to java.sql.Time - String strTime = fieldValue.getStringValue(); - String[] timeSplt = strTime.split(":"); - if (timeSplt.length != 3) { - throw new SQLException("Can not parse the value " + strTime + " to java.sql.Time"); - } - int hr = Integer.parseInt(timeSplt[0]); - int min = Integer.parseInt(timeSplt[1]); - int sec = 0, nanoSec = 0; - if (timeSplt[2].contains(".")) { - String[] secSplt = timeSplt[2].split("\\."); - sec = Integer.parseInt(secSplt[0]); - nanoSec = Integer.parseInt(secSplt[1]); - } else { - sec = Integer.parseInt(timeSplt[2]); - } - // LocalTime.of() - return Time.valueOf(LocalTime.of(hr, min, sec, nanoSec)); + return getTime(fieldValue); + } + return null; // TODO: Implementation for Arrow + } + + @Override + public Time getTime(int columnIndex) throws SQLException { + if (cursor == null) { + return null; // if the value is SQL NULL, the value returned is null + } else if (cursor instanceof FieldValueList) { + FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); + return getTime(fieldValue); + } + return null; // TODO: Implementation for Arrow + } + + private Time getTime(FieldValue fieldValue) throws SQLException { + if (fieldValue.getValue() != null) { + // Time ranges from 00:00:00 to 23:59:59.99999. in BigQuery. Parsing it to java.sql.Time + String strTime = fieldValue.getStringValue(); + String[] timeSplt = strTime.split(":"); + if (timeSplt.length != 3) { + throw new SQLException("Can not parse the value " + strTime + " to java.sql.Time"); + } + int hr = Integer.parseInt(timeSplt[0]); + int min = Integer.parseInt(timeSplt[1]); + int sec = 0, nanoSec = 0; + if (timeSplt[2].contains(".")) { + String[] secSplt = timeSplt[2].split("\\."); + sec = Integer.parseInt(secSplt[0]); + nanoSec = Integer.parseInt(secSplt[1]); } else { - return null; + sec = Integer.parseInt(timeSplt[2]); } + return Time.valueOf(LocalTime.of(hr, min, sec, nanoSec)); + } else { + return null; } - return null; // TODO: Implementation for Arrow } @Override @@ -265,5 +390,16 @@ public Date getDate(String fieldName) throws SQLException { } return null; // TODO: Implementation for Arrow } + + @Override + public Date getDate(int columnIndex) throws SQLException { + if (cursor == null) { + return null; // if the value is SQL NULL, the value returned is null + } else if (cursor instanceof FieldValueList) { + FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); + return fieldValue.getValue() == null ? null : Date.valueOf(fieldValue.getStringValue()); + } + return null; // TODO: Implementation for Arrow + } } } From d09b43c84332cebbfcdace977592eb5357b1082e Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Mon, 18 Oct 2021 13:04:25 +0530 Subject: [PATCH 047/277] Implemented testExecuteQuerySinglePageTableRowColInd for testing the columnIndex overloads --- .../cloud/bigquery/it/ITBigQueryTest.java | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index edcfa77f8..076c83259 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -2279,6 +2279,72 @@ public void testExecuteQuerySinglePageTableRow() throws SQLException { assertFalse(rs.next()); // no 3rd row in the table } + @Test + public void testExecuteQuerySinglePageTableRowColInd() throws SQLException { + String query = + "select StringField, BigNumericField, BooleanField, BytesField, IntegerField, TimestampField, FloatField, " + + "NumericField, TimeField, DateField, DateTimeField , GeographyField, RecordField.BytesField, RecordField.BooleanField, IntegerArrayField from " + + TABLE_ID_FASTQUERY_BQ_RESULTSET.getTable() + + " order by TimestampField"; + /* + Column Index mapping for ref: + StringField, 0 BigNumericField, 1 BooleanField, 2 BytesField, 3 IntegerField, 4 TimestampField, 5 FloatField, " 6 + NumericField, 7 TimeField, 8 DateField, 9 DateTimeField , 10 GeographyField, 11 RecordField.BytesField, 12 RecordField.BooleanField, 13 IntegerArrayField 14 + */ + ConnectionSettings connectionSettings = + ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); + Connection connection = bigquery.createConnection(connectionSettings); + BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); + ResultSet rs = bigQueryResultSet.getResultSet(); + Schema sc = bigQueryResultSet.getSchema(); + + assertEquals(BQ_RESULTSET_EXPECTED_SCHEMA, sc); // match the schema + assertEquals(2, bigQueryResultSet.getTotalRows()); // Expecting 2 rows + while (rs.next()) { + assertEquals(rs.getString(0), rs.getString("StringField")); + assertTrue(rs.getDouble(1) == rs.getDouble("BigNumericField")); + assertEquals(rs.getBoolean(2), rs.getBoolean("BooleanField")); + if (rs.getBytes(3) == null) { // both overloads should be null + assertEquals(rs.getBytes(3), rs.getBytes("BytesField")); + } else { // value in String representation should be the same + assertEquals( + new String(rs.getBytes(3), StandardCharsets.UTF_8), + new String(rs.getBytes("BytesField"), StandardCharsets.UTF_8)); + } + assertEquals(rs.getInt(4), rs.getInt("IntegerField")); + assertEquals(rs.getTimestamp(5), rs.getTimestamp("TimestampField")); + assertEquals(rs.getDate(9), rs.getDate("DateField")); + assertTrue(rs.getDouble("FloatField") == rs.getDouble(6)); + assertTrue(rs.getDouble("NumericField") == rs.getDouble(7)); + assertEquals(rs.getTime(8), rs.getTime("TimeField")); + assertEquals(rs.getString(10), rs.getString("DateTimeField")); + assertEquals(rs.getString(11), rs.getString("GeographyField")); + if (rs.getBytes(12) == null) { // both overloads should be null + assertEquals(rs.getBytes(12), rs.getBytes("BytesField_1")); + } else { // value in String representation should be the same + assertEquals( + new String(rs.getBytes(12), StandardCharsets.UTF_8), + new String(rs.getBytes("BytesField_1"), StandardCharsets.UTF_8)); + } + assertEquals(rs.getBoolean(13), rs.getBoolean("BooleanField_1")); + assertTrue( + rs.getObject("IntegerArrayField") instanceof com.google.cloud.bigquery.FieldValueList); + FieldValueList integerArrayFieldValue = + (com.google.cloud.bigquery.FieldValueList) rs.getObject("IntegerArrayField"); + assertTrue(rs.getObject(14) instanceof com.google.cloud.bigquery.FieldValueList); + FieldValueList integerArrayFieldValueColInd = + (com.google.cloud.bigquery.FieldValueList) rs.getObject(14); + assertEquals( + integerArrayFieldValue.size(), + integerArrayFieldValueColInd.size()); // Array has 4 elements + if (integerArrayFieldValue.size() == 4) { // as we are picking the third index + assertEquals( + (integerArrayFieldValue.get(2).getNumericValue()).intValue(), + (integerArrayFieldValueColInd.get(2).getNumericValue()).intValue()); + } + } + } + @Test public void testExecuteSelectStruct() throws SQLException { String query = "select (STRUCT(\"Vancouver\" as city, 5 as years)) as address"; From b441215934927ab26867e33023fadac9ce176919 Mon Sep 17 00:00:00 2001 From: stephwang Date: Mon, 18 Oct 2021 14:45:42 -0400 Subject: [PATCH 048/277] update test case for struct --- .../google/cloud/bigquery/ConnectionImpl.java | 2 ++ .../cloud/bigquery/it/ITBigQueryTest.java | 19 ++++++++++++++----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 7e000d385..4fcf48fb1 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -168,6 +168,7 @@ private BigQueryResultSet processQueryResponseResults( Iterable fieldValueLists = getIterableFieldValueList(results.getRows(), schema); // Producer thread for populating the buffer row by row + // TODO: Update to use a configurable number of threads (default 4) to populate the producer BlockingQueue> buffer = new LinkedBlockingDeque<>(1000); // TODO: Update the capacity. Prefetch limit Runnable populateBufferRunnable = @@ -335,6 +336,7 @@ private BigQueryResultSet processGetQueryResults(JobId jobId, GetQueryResultsRes } private boolean isFastQuerySupported() { + // TODO: add regex logic to check for scripting return connectionSettings.getClustering() == null && connectionSettings.getCreateDisposition() == null && connectionSettings.getDestinationEncryptionConfiguration() == null diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 076c83259..c19e439ba 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -2355,15 +2355,24 @@ public void testExecuteSelectStruct() throws SQLException { assertEquals(1, bigQueryResultSet.getTotalRows()); Schema schema = bigQueryResultSet.getSchema(); - assertEquals(schema.getFields().get(0).getName(), "address"); - assertEquals(schema.getFields().get(0).getMode(), Field.Mode.NULLABLE); + assertEquals("address", schema.getFields().get(0).getName()); + assertEquals(Field.Mode.NULLABLE, schema.getFields().get(0).getMode()); // Backend is currently returning RECORD which is LegacySQLTypeName. - assertEquals(schema.getFields().get(0).getType(), LegacySQLTypeName.RECORD); + assertEquals(LegacySQLTypeName.RECORD, schema.getFields().get(0).getType()); + assertEquals("city", schema.getFields().get(0).getSubFields().get(0).getName()); + assertEquals( + LegacySQLTypeName.STRING, schema.getFields().get(0).getSubFields().get(0).getType()); + assertEquals("years", schema.getFields().get(0).getSubFields().get(1).getName()); + assertEquals( + LegacySQLTypeName.INTEGER, schema.getFields().get(0).getSubFields().get(1).getType()); ResultSet rs = bigQueryResultSet.getResultSet(); assertTrue(rs.next()); - // TODO: Figure out what this is supposed to return - // Object o = rs.getObject("address"); // is this supposed to return the reinterpreted JSON? + FieldValueList addressFieldValue = + (com.google.cloud.bigquery.FieldValueList) rs.getObject("address"); + assertEquals(rs.getObject("address"), rs.getObject(0)); + assertEquals("Vancouver", addressFieldValue.get(0).getStringValue()); + assertEquals("5", addressFieldValue.get(1).getStringValue()); assertFalse(rs.next()); // only 1 row of data } From 3f2a1288d6943e52558f94052c015374ef21949a Mon Sep 17 00:00:00 2001 From: stephwang Date: Tue, 19 Oct 2021 11:59:57 -0400 Subject: [PATCH 049/277] add more test cases for nested data types --- .../cloud/bigquery/it/ITBigQueryTest.java | 99 ++++++++++++++++++- 1 file changed, 95 insertions(+), 4 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index c19e439ba..64a9ee787 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -64,6 +64,7 @@ import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.FieldList; import com.google.cloud.bigquery.FieldValue; +import com.google.cloud.bigquery.FieldValue.Attribute; import com.google.cloud.bigquery.FieldValueList; import com.google.cloud.bigquery.FormatOptions; import com.google.cloud.bigquery.HivePartitioningOptions; @@ -2220,7 +2221,7 @@ public void testQueryExternalHivePartitioningOptionCustomLayout() throws Interru } @Test - public void testExecuteQuerySinglePageTableRow() throws SQLException { + public void testExecuteSelectSinglePageTableRow() throws SQLException { String query = "select StringField, BigNumericField, BooleanField, BytesField, IntegerField, TimestampField, FloatField, " + "NumericField, TimeField, DateField, DateTimeField , GeographyField, RecordField.BytesField, RecordField.BooleanField, IntegerArrayField from " @@ -2280,7 +2281,7 @@ public void testExecuteQuerySinglePageTableRow() throws SQLException { } @Test - public void testExecuteQuerySinglePageTableRowColInd() throws SQLException { + public void testExecuteSelectSinglePageTableRowColInd() throws SQLException { String query = "select StringField, BigNumericField, BooleanField, BytesField, IntegerField, TimestampField, FloatField, " + "NumericField, TimeField, DateField, DateTimeField , GeographyField, RecordField.BytesField, RecordField.BooleanField, IntegerArrayField from " @@ -2357,14 +2358,16 @@ public void testExecuteSelectStruct() throws SQLException { Schema schema = bigQueryResultSet.getSchema(); assertEquals("address", schema.getFields().get(0).getName()); assertEquals(Field.Mode.NULLABLE, schema.getFields().get(0).getMode()); - // Backend is currently returning RECORD which is LegacySQLTypeName. + // Backend is currently returning LegacySQLTypeName. Tracking bug: b/202977620 assertEquals(LegacySQLTypeName.RECORD, schema.getFields().get(0).getType()); assertEquals("city", schema.getFields().get(0).getSubFields().get(0).getName()); assertEquals( LegacySQLTypeName.STRING, schema.getFields().get(0).getSubFields().get(0).getType()); + assertEquals(Field.Mode.NULLABLE, schema.getFields().get(0).getSubFields().get(0).getMode()); assertEquals("years", schema.getFields().get(0).getSubFields().get(1).getName()); assertEquals( LegacySQLTypeName.INTEGER, schema.getFields().get(0).getSubFields().get(1).getType()); + assertEquals(Field.Mode.NULLABLE, schema.getFields().get(0).getSubFields().get(1).getMode()); ResultSet rs = bigQueryResultSet.getResultSet(); assertTrue(rs.next()); @@ -2372,10 +2375,98 @@ public void testExecuteSelectStruct() throws SQLException { (com.google.cloud.bigquery.FieldValueList) rs.getObject("address"); assertEquals(rs.getObject("address"), rs.getObject(0)); assertEquals("Vancouver", addressFieldValue.get(0).getStringValue()); - assertEquals("5", addressFieldValue.get(1).getStringValue()); + assertEquals(5, addressFieldValue.get(1).getLongValue()); assertFalse(rs.next()); // only 1 row of data } + @Test + public void testExecuteSelectStructSubField() throws SQLException { + String query = + "select address.city from (select (STRUCT(\"Vancouver\" as city, 5 as years)) as address)"; + ConnectionSettings connectionSettings = + ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); + Connection connection = bigquery.createConnection(connectionSettings); + BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); + assertEquals(1, bigQueryResultSet.getTotalRows()); + + Schema schema = bigQueryResultSet.getSchema(); + assertEquals("city", schema.getFields().get(0).getName()); + assertEquals(Field.Mode.NULLABLE, schema.getFields().get(0).getMode()); + // Backend is currently returning LegacySQLTypeName. Tracking bug: b/202977620 + assertEquals(LegacySQLTypeName.STRING, schema.getFields().get(0).getType()); + assertNull( + schema.getFields().get(0).getSubFields()); // this is a String field without any subfields + + ResultSet rs = bigQueryResultSet.getResultSet(); + assertTrue(rs.next()); + String cityFieldValue = rs.getString("city"); + assertEquals(rs.getString("city"), rs.getObject(0)); + assertEquals("Vancouver", cityFieldValue); + assertFalse(rs.next()); // only 1 row of data + } + + @Test + public void testExecuteSelectArray() throws SQLException { + String query = "SELECT [1,2,3]"; + ConnectionSettings connectionSettings = + ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); + Connection connection = bigquery.createConnection(connectionSettings); + BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); + assertEquals(1, bigQueryResultSet.getTotalRows()); + + Schema schema = bigQueryResultSet.getSchema(); + assertEquals("f0_", schema.getFields().get(0).getName()); + assertEquals(Field.Mode.REPEATED, schema.getFields().get(0).getMode()); + assertEquals(LegacySQLTypeName.INTEGER, schema.getFields().get(0).getType()); + assertNull(schema.getFields().get(0).getSubFields()); // no subfields for Integers + + ResultSet rs = bigQueryResultSet.getResultSet(); + assertTrue(rs.next()); + FieldValueList arrayFieldValue = (com.google.cloud.bigquery.FieldValueList) rs.getObject(0); + assertEquals(1, arrayFieldValue.get(0).getLongValue()); + assertEquals(2, arrayFieldValue.get(1).getLongValue()); + assertEquals(3, arrayFieldValue.get(2).getLongValue()); + } + + @Test + public void testExecuteSelectArrayOfStruct() throws SQLException { + String query = + "SELECT [STRUCT(\"Vancouver\" as city, 5 as years), STRUCT(\"Boston\" as city, 10 as years)]"; + ConnectionSettings connectionSettings = + ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); + Connection connection = bigquery.createConnection(connectionSettings); + BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); + assertEquals(1, bigQueryResultSet.getTotalRows()); + + Schema schema = bigQueryResultSet.getSchema(); + assertEquals("f0_", schema.getFields().get(0).getName()); + assertEquals(Field.Mode.REPEATED, schema.getFields().get(0).getMode()); + // Backend is currently returning LegacySQLTypeName. Tracking bug: b/202977620 + // Verify the field metadata of the two subfields of the struct + assertEquals(LegacySQLTypeName.RECORD, schema.getFields().get(0).getType()); + assertEquals("city", schema.getFields().get(0).getSubFields().get(0).getName()); + assertEquals( + LegacySQLTypeName.STRING, schema.getFields().get(0).getSubFields().get(0).getType()); + assertEquals(Field.Mode.NULLABLE, schema.getFields().get(0).getSubFields().get(0).getMode()); + assertEquals("years", schema.getFields().get(0).getSubFields().get(1).getName()); + assertEquals( + LegacySQLTypeName.INTEGER, schema.getFields().get(0).getSubFields().get(1).getType()); + assertEquals(Field.Mode.NULLABLE, schema.getFields().get(0).getSubFields().get(1).getMode()); + + ResultSet rs = bigQueryResultSet.getResultSet(); + assertTrue(rs.next()); + FieldValueList arrayOfStructFieldValue = + (com.google.cloud.bigquery.FieldValueList) rs.getObject(0); + // Verify the values of the two structs in the array + assertEquals(Attribute.RECORD, arrayOfStructFieldValue.get(0).getAttribute()); + assertEquals( + "Vancouver", arrayOfStructFieldValue.get(0).getRecordValue().get(0).getStringValue()); + assertEquals(5, arrayOfStructFieldValue.get(0).getRecordValue().get(1).getLongValue()); + assertEquals(Attribute.RECORD, arrayOfStructFieldValue.get(1).getAttribute()); + assertEquals("Boston", arrayOfStructFieldValue.get(1).getRecordValue().get(0).getStringValue()); + assertEquals(10, arrayOfStructFieldValue.get(1).getRecordValue().get(1).getLongValue()); + } + @Test public void testFastQueryMultipleRuns() throws InterruptedException { String query = From 1847a91385ef5c8233c48613419a0cb88c310b4c Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 26 Oct 2021 18:25:50 +0530 Subject: [PATCH 050/277] updated listTableDataWithRowLimit --- .../main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java index f09befd18..d19da3171 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java @@ -259,7 +259,7 @@ TableDataList listTableData( * @throws BigQueryException upon failure */ TableDataList listTableDataWithRowLimit( - String projectId, String datasetId, String tableId, Long rowLimit); + String projectId, String datasetId, String tableId, Long rowLimit, String pageToken); /** * Returns the requested job or {@code null} if not found. From 6fc492c900cdf7aa0a59b8dd6dee8861448df148 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 26 Oct 2021 18:27:58 +0530 Subject: [PATCH 051/277] Added parameter for TableDataList.listTableDataWithRowLimit --- google-cloud-bigquery/clirr-ignored-differences.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-bigquery/clirr-ignored-differences.xml b/google-cloud-bigquery/clirr-ignored-differences.xml index 832712adb..d349e0863 100644 --- a/google-cloud-bigquery/clirr-ignored-differences.xml +++ b/google-cloud-bigquery/clirr-ignored-differences.xml @@ -25,6 +25,6 @@ 7012 com/google/cloud/bigquery/spi/v2/BigQueryRpc - com.google.api.services.bigquery.model.TableDataList listTableDataWithRowLimit(java.lang.String, java.lang.String, java.lang.String, java.lang.Long) + com.google.api.services.bigquery.model.TableDataList listTableDataWithRowLimit(java.lang.String, java.lang.String, java.lang.String, java.lang.Long, java.lang.String) \ No newline at end of file From 50843fe162645f789bb734ee1667b73dc3de29a4 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 26 Oct 2021 18:28:51 +0530 Subject: [PATCH 052/277] Modified listTableDataWithRowLimit to include pageToken --- .../com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java index 620b9223e..2b85a966a 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java @@ -530,13 +530,18 @@ public TableDataList listTableData( @Override public TableDataList listTableDataWithRowLimit( - String projectId, String datasetId, String tableId, Long preFetchedRowLimit) { + String projectId, + String datasetId, + String tableId, + Long preFetchedRowLimit, + String pageToken) { try { return bigquery .tabledata() .list(projectId, datasetId, tableId) .setPrettyPrint(false) .setMaxResults(preFetchedRowLimit) + .setPageToken(pageToken) .execute(); } catch (IOException ex) { throw translate(ex); From 41279ff6261ed2fddaea200193b64b2f3fc51f96 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 26 Oct 2021 18:29:44 +0530 Subject: [PATCH 053/277] Modified processQueryResponseResults to include pagination logic --- .../google/cloud/bigquery/ConnectionImpl.java | 72 +++++++++++-------- 1 file changed, 42 insertions(+), 30 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 4fcf48fb1..671aa1d4e 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -120,6 +120,7 @@ private BigQueryResultSet queryRpc(final String projectId, final QueryRequest qu } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); } + if (results.getErrors() != null) { List bigQueryErrors = results.getErrors().stream() @@ -165,42 +166,59 @@ private BigQueryResultSet processQueryResponseResults( schema = Schema.fromPb(results.getSchema()); numRows = results.getTotalRows().longValue(); - Iterable fieldValueLists = getIterableFieldValueList(results.getRows(), schema); - // Producer thread for populating the buffer row by row // TODO: Update to use a configurable number of threads (default 4) to populate the producer BlockingQueue> buffer = new LinkedBlockingDeque<>(1000); // TODO: Update the capacity. Prefetch limit + Runnable populateBufferRunnable = () -> { // producer thread populating the buffer - // List tableRows = results.getRows(); - for (FieldValueList fieldValueList : - fieldValueLists) { // TODO: Handle the logic of pagination - try { - buffer.put(fieldValueList); - } catch (InterruptedException e) { - e.printStackTrace(); + Iterable fieldValueLists = + getIterableFieldValueList(results.getRows(), schema); + + boolean hasRows = true; // as we have to process the first page + String pageToken = results.getPageToken(); + while (hasRows) { + + for (FieldValueList fieldValueList : fieldValueLists) { + try { + buffer.put(fieldValueList); + } catch (InterruptedException e) { + e.printStackTrace(); + } } - } - if (results.getPageToken() == null) { - buffer.offer( - new EndOfFieldValueList()); // A Hack to inform the BQResultSet that the blocking - // queue won't be populated with more records + if (pageToken != null) { // request next page + TableDataList tabledataList = + tableDataListRpc( + connectionSettings.getDestinationTable(), + pageToken); // TODO: It appears connectionSettings.getDestinationTable() is + // null, check how can be pass the required params + fieldValueLists = getIterableFieldValueList(tabledataList.getRows(), schema); + ; // get next set of values + hasRows = true; // new page was requested, which is not yet processed + pageToken = tabledataList.getPageToken(); // next page's token + } else { // the current page has been processed and there's no next page + hasRows = false; + } } + buffer.offer( + new EndOfFieldValueList()); // All the pages has been processed, put this marker }; Thread populateBufferWorker = new Thread(populateBufferRunnable); populateBufferWorker.start(); // child process to populate the buffer - // only 1 page of result - if (results.getPageToken() == null) { - return new BigQueryResultSetImpl>(schema, numRows, buffer); - } + // if (results.getPageToken() == null) { + return new BigQueryResultSetImpl>(schema, numRows, buffer); + // } + + /* + // TODO: This part is done above, will remove the commented code in the next commit // use tabledata.list or Read API to fetch subsequent pages of results long totalRows = results.getTotalRows().longValue(); long pageRows = results.getRows().size(); JobId jobId = JobId.fromPb(results.getJobReference()); - return null; // TODO - Implement getQueryResultsWithJobId(totalRows, pageRows, schema, jobId); + return null; // TODO - Implement getQueryResultsWithJobId(totalRows, pageRows, schema, jobId);*/ } /* Returns query results using either tabledata.list or the high throughput Read API */ @@ -209,7 +227,7 @@ private BigQueryResultSet getQueryResultsWithJobId( TableId destinationTable = queryJobsGetRpc(jobId); return useReadAPI(totalRows, pageRows) ? highThroughPutRead(destinationTable) - : tableDataListRpc(destinationTable, schema); + : null; // TODO - plugin tableDataListRpc(destinationTable, schema, null); } /* Returns the destinationTable from jobId by calling jobs.get API */ @@ -243,14 +261,13 @@ private TableId queryJobsGetRpc(JobId jobId) { return ((QueryJobConfiguration) job.getConfiguration()).getDestinationTable(); } - private BigQueryResultSet tableDataListRpc(TableId destinationTable, Schema schema) { + private TableDataList tableDataListRpc(TableId destinationTable, String pageToken) { try { final TableId completeTableId = destinationTable.setProjectId( Strings.isNullOrEmpty(destinationTable.getProject()) ? bigQueryOptions.getProjectId() : destinationTable.getProject()); - TableDataList results = runWithRetries( () -> @@ -260,18 +277,13 @@ private BigQueryResultSet tableDataListRpc(TableId destinationTable, Schema sche completeTableId.getProject(), completeTableId.getDataset(), completeTableId.getTable(), - connectionSettings.getNumBufferedRows()), + connectionSettings.getNumBufferedRows(), + pageToken), bigQueryOptions.getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, bigQueryOptions.getClock()); - long numRows = results.getTotalRows() == null ? 0 : results.getTotalRows().longValue(); - /* return new BigQueryResultSetImpl( - schema, numRows, null - */ - /* TODO: iterate(pagesAfterFirstPage) */ - /* );*/ - return null; // TODO: Plugin the buffer logic + return results; } catch (RetryHelper.RetryHelperException e) { throw BigQueryException.translateAndThrow(e); } From ca33f250dcf9c9bb867f967922c46a05ea7412fd Mon Sep 17 00:00:00 2001 From: stephwang Date: Tue, 26 Oct 2021 11:56:30 -0400 Subject: [PATCH 054/277] manage dependencies --- google-cloud-bigquery/pom.xml | 35 ++++++++++++------- .../google/cloud/bigquery/ConnectionImpl.java | 2 +- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/google-cloud-bigquery/pom.xml b/google-cloud-bigquery/pom.xml index 88ea25fa4..d0e2a1d43 100644 --- a/google-cloud-bigquery/pom.xml +++ b/google-cloud-bigquery/pom.xml @@ -29,18 +29,6 @@ com.google.http-client google-http-client-jackson2 - - com.google.cloud - google-cloud-datacatalog - - - com.google.api.grpc - proto-google-cloud-datacatalog-v1 - - - com.google.cloud - google-cloud-storage - com.google.auto.value auto-value-annotations @@ -97,6 +85,21 @@ + + com.google.cloud + google-cloud-datacatalog + test + + + com.google.api.grpc + proto-google-cloud-datacatalog-v1 + test + + + com.google.cloud + google-cloud-storage + test + junit junit @@ -105,6 +108,7 @@ com.google.truth truth + test org.mockito @@ -116,6 +120,13 @@ assertj-core test + + + org.checkerframework + checker-qual + 3.12.0 + test + diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 671aa1d4e..7e25ef3cd 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -193,7 +193,7 @@ private BigQueryResultSet processQueryResponseResults( tableDataListRpc( connectionSettings.getDestinationTable(), pageToken); // TODO: It appears connectionSettings.getDestinationTable() is - // null, check how can be pass the required params + // null, check how can be pass the required params fieldValueLists = getIterableFieldValueList(tabledataList.getRows(), schema); ; // get next set of values hasRows = true; // new page was requested, which is not yet processed From a5b77345d3ebf45ee093a032bdfac80fc3365e85 Mon Sep 17 00:00:00 2001 From: stephwang Date: Tue, 26 Oct 2021 14:02:36 -0400 Subject: [PATCH 055/277] manage dependencies --- google-cloud-bigquery/pom.xml | 7 ------- pom.xml | 7 ------- 2 files changed, 14 deletions(-) diff --git a/google-cloud-bigquery/pom.xml b/google-cloud-bigquery/pom.xml index d0e2a1d43..1361f7324 100644 --- a/google-cloud-bigquery/pom.xml +++ b/google-cloud-bigquery/pom.xml @@ -120,13 +120,6 @@ assertj-core test - - - org.checkerframework - checker-qual - 3.12.0 - test - diff --git a/pom.xml b/pom.xml index 44746cc4d..59d92fc63 100644 --- a/pom.xml +++ b/pom.xml @@ -129,13 +129,6 @@ 1.5.9 test - - - org.checkerframework - checker-qual - 3.8.0 - test - org.assertj assertj-core From 5ee68b48a3e99e7548a75a68c2661117d9fbbba8 Mon Sep 17 00:00:00 2001 From: stephwang Date: Tue, 26 Oct 2021 20:04:04 -0400 Subject: [PATCH 056/277] update pagination logic --- .../google/cloud/bigquery/ConnectionImpl.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 7e25ef3cd..5f78ac04c 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -131,11 +131,13 @@ private BigQueryResultSet queryRpc(final String projectId, final QueryRequest qu throw new BigQueryException(bigQueryErrors); } + // Query finished running and we can paginate all the results if (results.getJobComplete() && results.getSchema() != null) { return processQueryResponseResults(results); } else { // Query is long running (> 10s) and hasn't completed yet, or query completed but didn't - // return the schema, fallback to GetQueryResults. Some operations don't return the schema and + // return the schema, fallback to jobs.insert path. Some operations don't return the schema + // and // can be optimized here, but this is left as future work. long totalRows = results.getTotalRows().longValue(); long pageRows = results.getRows().size(); @@ -189,13 +191,11 @@ private BigQueryResultSet processQueryResponseResults( } if (pageToken != null) { // request next page - TableDataList tabledataList = - tableDataListRpc( - connectionSettings.getDestinationTable(), - pageToken); // TODO: It appears connectionSettings.getDestinationTable() is - // null, check how can be pass the required params + JobId jobId = JobId.fromPb(results.getJobReference()); + TableId destinationTable = queryJobsGetRpc(jobId); + TableDataList tabledataList = tableDataListRpc(destinationTable, pageToken); fieldValueLists = getIterableFieldValueList(tabledataList.getRows(), schema); - ; // get next set of values + // get next set of values hasRows = true; // new page was requested, which is not yet processed pageToken = tabledataList.getPageToken(); // next page's token } else { // the current page has been processed and there's no next page @@ -208,9 +208,8 @@ private BigQueryResultSet processQueryResponseResults( Thread populateBufferWorker = new Thread(populateBufferRunnable); populateBufferWorker.start(); // child process to populate the buffer - // if (results.getPageToken() == null) { + // Only 1 page of results return new BigQueryResultSetImpl>(schema, numRows, buffer); - // } /* // TODO: This part is done above, will remove the commented code in the next commit From ac3391565b7dba0439f8e4a4462cca4c14fac79f Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 28 Oct 2021 08:53:54 +0530 Subject: [PATCH 057/277] Minor cleanup --- .../google/cloud/bigquery/ConnectionImpl.java | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 5f78ac04c..ae108ef2f 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -142,7 +142,7 @@ private BigQueryResultSet queryRpc(final String projectId, final QueryRequest qu long totalRows = results.getTotalRows().longValue(); long pageRows = results.getRows().size(); JobId jobId = JobId.fromPb(results.getJobReference()); - return null; // TODO getQueryResultsWithJobId(totalRows, pageRows, null, jobId); + return getQueryResultsWithJobId(totalRows, pageRows, null, jobId); } } @@ -170,6 +170,7 @@ private BigQueryResultSet processQueryResponseResults( // Producer thread for populating the buffer row by row // TODO: Update to use a configurable number of threads (default 4) to populate the producer + // TODO: Have just one child process at most, instead of the thread BlockingQueue> buffer = new LinkedBlockingDeque<>(1000); // TODO: Update the capacity. Prefetch limit @@ -193,9 +194,9 @@ private BigQueryResultSet processQueryResponseResults( if (pageToken != null) { // request next page JobId jobId = JobId.fromPb(results.getJobReference()); TableId destinationTable = queryJobsGetRpc(jobId); + // get next set of values TableDataList tabledataList = tableDataListRpc(destinationTable, pageToken); fieldValueLists = getIterableFieldValueList(tabledataList.getRows(), schema); - // get next set of values hasRows = true; // new page was requested, which is not yet processed pageToken = tabledataList.getPageToken(); // next page's token } else { // the current page has been processed and there's no next page @@ -208,25 +209,20 @@ private BigQueryResultSet processQueryResponseResults( Thread populateBufferWorker = new Thread(populateBufferRunnable); populateBufferWorker.start(); // child process to populate the buffer - // Only 1 page of results + // This will work for pagination as well, as buffer is getting updated asynchronously return new BigQueryResultSetImpl>(schema, numRows, buffer); - - /* - // TODO: This part is done above, will remove the commented code in the next commit - // use tabledata.list or Read API to fetch subsequent pages of results - long totalRows = results.getTotalRows().longValue(); - long pageRows = results.getRows().size(); - JobId jobId = JobId.fromPb(results.getJobReference()); - return null; // TODO - Implement getQueryResultsWithJobId(totalRows, pageRows, schema, jobId);*/ } /* Returns query results using either tabledata.list or the high throughput Read API */ private BigQueryResultSet getQueryResultsWithJobId( long totalRows, long pageRows, Schema schema, JobId jobId) { TableId destinationTable = queryJobsGetRpc(jobId); - return useReadAPI(totalRows, pageRows) - ? highThroughPutRead(destinationTable) - : null; // TODO - plugin tableDataListRpc(destinationTable, schema, null); + + return null; // use processQueryResponseResults(); for remaining pages + /* return useReadAPI(totalRows, pageRows) + ? highThroughPutRead(destinationTable) + : null; // plugin tableDataListRpc(destinationTable, schema, null);, Use processQueryResponseResults ? + */ } /* Returns the destinationTable from jobId by calling jobs.get API */ @@ -293,7 +289,8 @@ private BigQueryResultSet highThroughPutRead(TableId destinationTable) { } /* Returns results of the query associated with the provided job using jobs.getQueryResults API */ - private BigQueryResultSet getQueryResultsRpc(JobId jobId) { + private BigQueryResultSet getQueryResultsRpc( + JobId jobId) { // TODO(prasmish) temp: This is a slower endpoint JobId completeJobId = jobId .setProjectId(bigQueryOptions.getProjectId()) From 427e82d15ed308f0e242e7f9903d6f8b80e25ae7 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 28 Oct 2021 12:53:14 +0530 Subject: [PATCH 058/277] Updated the bufferSize --- .../java/com/google/cloud/bigquery/ConnectionImpl.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index ae108ef2f..6e9cfdf24 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -171,8 +171,13 @@ private BigQueryResultSet processQueryResponseResults( // Producer thread for populating the buffer row by row // TODO: Update to use a configurable number of threads (default 4) to populate the producer // TODO: Have just one child process at most, instead of the thread - BlockingQueue> buffer = - new LinkedBlockingDeque<>(1000); // TODO: Update the capacity. Prefetch limit + // Keeping the buffersize more than the page size and ensuring it's always a reasonable number + int bufferSize = + (int) + (connectionSettings.getNumBufferedRows() >= 10000 + ? (connectionSettings.getNumBufferedRows() * 2) + : 20000); + BlockingQueue> buffer = new LinkedBlockingDeque<>(bufferSize); Runnable populateBufferRunnable = () -> { // producer thread populating the buffer @@ -190,7 +195,6 @@ private BigQueryResultSet processQueryResponseResults( e.printStackTrace(); } } - if (pageToken != null) { // request next page JobId jobId = JobId.fromPb(results.getJobReference()); TableId destinationTable = queryJobsGetRpc(jobId); From 90a238d91dfb7db857ef0ddb21256eb963cafa88 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 28 Oct 2021 12:53:46 +0530 Subject: [PATCH 059/277] Added testBQResultSetPagination --- .../cloud/bigquery/it/ITBigQueryTest.java | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 804f2c09f..4a60b5521 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -2280,6 +2280,31 @@ public void testExecuteSelectSinglePageTableRow() throws SQLException { assertFalse(rs.next()); // no 3rd row in the table } + @Test + public void testBQResultSetPagination() throws SQLException { + // TODO(prasmish) find a way to auto populate large dataset for testing pagination (Eg: upload a + // CSV) and update the query + String query = + "SELECT pickup_datetime, passenger_count, store_and_fwd_flag FROM `java-docs-samples-testing.bigquery_test_dataset.tlc_yellow_trips_2017_stephwang` " + + "where pickup_datetime is not null and store_and_fwd_flag is not null order by pickup_datetime limit 200000"; + ConnectionSettings connectionSettings = + ConnectionSettings.newBuilder() + .setDefaultDataset(DatasetId.of("bigquery_test_dataset")) + .setNumBufferedRows(10000L) // page size + .build(); + Connection connection = bigquery.createConnection(connectionSettings); + BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); + ResultSet rs = bigQueryResultSet.getResultSet(); + int cnt = 0; + while (rs.next()) { // pagination starts after approx 120,000 records + assertNotNull(rs.getString(0)); + assertTrue(rs.getInt(1) >= 0); + assertNotNull(rs.getString(2)); + ++cnt; + } + assertEquals(200000, cnt); // total 200000 rows should be read + } + @Test public void testExecuteSelectSinglePageTableRowColInd() throws SQLException { String query = From b1dd5985880624fe137f91a0b8cc37073394bc31 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 28 Oct 2021 13:40:01 +0530 Subject: [PATCH 060/277] refactored processQueryResponseResults --- .../com/google/cloud/bigquery/ConnectionImpl.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 6e9cfdf24..3c7d73c2f 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -174,9 +174,10 @@ private BigQueryResultSet processQueryResponseResults( // Keeping the buffersize more than the page size and ensuring it's always a reasonable number int bufferSize = (int) - (connectionSettings.getNumBufferedRows() >= 10000 - ? (connectionSettings.getNumBufferedRows() * 2) - : 20000); + (connectionSettings.getNumBufferedRows() == null + || connectionSettings.getNumBufferedRows() < 10000 + ? 20000 + : (connectionSettings.getNumBufferedRows() * 2)); BlockingQueue> buffer = new LinkedBlockingDeque<>(bufferSize); Runnable populateBufferRunnable = @@ -186,8 +187,9 @@ private BigQueryResultSet processQueryResponseResults( boolean hasRows = true; // as we have to process the first page String pageToken = results.getPageToken(); + JobId jobId = JobId.fromPb(results.getJobReference()); + final TableId destinationTable = queryJobsGetRpc(jobId); while (hasRows) { - for (FieldValueList fieldValueList : fieldValueLists) { try { buffer.put(fieldValueList); @@ -196,8 +198,6 @@ private BigQueryResultSet processQueryResponseResults( } } if (pageToken != null) { // request next page - JobId jobId = JobId.fromPb(results.getJobReference()); - TableId destinationTable = queryJobsGetRpc(jobId); // get next set of values TableDataList tabledataList = tableDataListRpc(destinationTable, pageToken); fieldValueLists = getIterableFieldValueList(tabledataList.getRows(), schema); From 12ad71c4951721c50cf10ce4116bd409bcefabd5 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 29 Oct 2021 10:29:47 +0530 Subject: [PATCH 061/277] updated testBQResultSetPagination to use generated table --- .../cloud/bigquery/it/ITBigQueryTest.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 4a60b5521..1afa94e86 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -2282,14 +2282,13 @@ public void testExecuteSelectSinglePageTableRow() throws SQLException { @Test public void testBQResultSetPagination() throws SQLException { - // TODO(prasmish) find a way to auto populate large dataset for testing pagination (Eg: upload a - // CSV) and update the query String query = - "SELECT pickup_datetime, passenger_count, store_and_fwd_flag FROM `java-docs-samples-testing.bigquery_test_dataset.tlc_yellow_trips_2017_stephwang` " - + "where pickup_datetime is not null and store_and_fwd_flag is not null order by pickup_datetime limit 200000"; + "SELECT date, county, state_name, confirmed_cases, deaths FROM " + + TABLE_ID_LARGE.getTable() + + " where date is not null and county is not null and state_name is not null order by date limit 300000"; ConnectionSettings connectionSettings = ConnectionSettings.newBuilder() - .setDefaultDataset(DatasetId.of("bigquery_test_dataset")) + .setDefaultDataset(DatasetId.of(DATASET)) .setNumBufferedRows(10000L) // page size .build(); Connection connection = bigquery.createConnection(connectionSettings); @@ -2297,12 +2296,14 @@ public void testBQResultSetPagination() throws SQLException { ResultSet rs = bigQueryResultSet.getResultSet(); int cnt = 0; while (rs.next()) { // pagination starts after approx 120,000 records - assertNotNull(rs.getString(0)); - assertTrue(rs.getInt(1) >= 0); + assertNotNull(rs.getDate(0)); + assertNotNull(rs.getString(1)); assertNotNull(rs.getString(2)); + assertTrue(rs.getInt(3) >= 0); + assertTrue(rs.getInt(4) >= 0); ++cnt; } - assertEquals(200000, cnt); // total 200000 rows should be read + assertEquals(300000, cnt); // total 300000 rows should be read } @Test From 7651405cb1cce1f38d0d841d05609a161c21d586 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 9 Nov 2021 16:20:18 +0530 Subject: [PATCH 062/277] implemented a child thread for fetching next page --- .../google/cloud/bigquery/ConnectionImpl.java | 83 ++++++++++++++++++- 1 file changed, 81 insertions(+), 2 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 3c7d73c2f..bd0536e7a 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -169,8 +169,87 @@ private BigQueryResultSet processQueryResponseResults( numRows = results.getTotalRows().longValue(); // Producer thread for populating the buffer row by row - // TODO: Update to use a configurable number of threads (default 4) to populate the producer - // TODO: Have just one child process at most, instead of the thread + // Keeping the buffersize more than the page size and ensuring it's always a reasonable number + int bufferSize = + (int) + (connectionSettings.getNumBufferedRows() == null + || connectionSettings.getNumBufferedRows() < 10000 + ? 20000 + : (connectionSettings.getNumBufferedRows() * 2)); + BlockingQueue> buffer = new LinkedBlockingDeque<>(bufferSize); + BlockingQueue nextPage = + new LinkedBlockingDeque<>(1); // we will be keeping atmost 1 page fetched upfront + + Runnable populateBufferRunnable = + () -> { // producer thread populating the buffer + Iterable fieldValueLists = + getIterableFieldValueList(results.getRows(), schema); + + boolean hasRows = true; // as we have to process the first page + String pageToken = results.getPageToken(); + JobId jobId = JobId.fromPb(results.getJobReference()); + final TableId destinationTable = queryJobsGetRpc(jobId); + while (hasRows) { + if (pageToken + != null) { // spawn a child thread to fetch the next page before the current page is + // processed + final String nextPageToken = pageToken; + Runnable nextPageTask = + () -> { + try { + TableDataList tabledataList = + tableDataListRpc(destinationTable, nextPageToken); + nextPage.put(tabledataList); + } catch (InterruptedException e) { + e.printStackTrace(); // TODO: Throw an exception back + } + }; + Thread nextPageFetcher = new Thread(nextPageTask); + nextPageFetcher.start(); + } + + for (FieldValueList fieldValueList : fieldValueLists) { + try { + buffer.put(fieldValueList); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + if (pageToken != null) { + // next page would have been requested using a child process, check if the it's + // available + try { + TableDataList tabledataList = nextPage.take(); + fieldValueLists = getIterableFieldValueList(tabledataList.getRows(), schema); + hasRows = true; // new page was requested, which is not yet processed + pageToken = tabledataList.getPageToken(); // next page's token + } catch (InterruptedException e) { + e.printStackTrace(); // TODO: Throw an exception back + } + } else { // the current page has been processed and there's no next page + hasRows = false; + } + } + buffer.offer( + new EndOfFieldValueList()); // All the pages has been processed, put this marker + }; + Thread populateBufferWorker = new Thread(populateBufferRunnable); + populateBufferWorker.start(); // producer process to populate the buffer + + // This will work for pagination as well, as buffer is getting updated asynchronously + return new BigQueryResultSetImpl>(schema, numRows, buffer); + } + + // TODO(prasmish): Remove it after benchmarking + @Deprecated + private BigQueryResultSet processQueryResponseResultsSingleThreadedProducer( + com.google.api.services.bigquery.model.QueryResponse results) { + Schema schema; + long numRows; + schema = Schema.fromPb(results.getSchema()); + numRows = results.getTotalRows().longValue(); + + // Producer thread for populating the buffer row by row // Keeping the buffersize more than the page size and ensuring it's always a reasonable number int bufferSize = (int) From 403c6ef523651bb8ae48970468ac52917a2fe13b Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 10 Nov 2021 14:02:01 +0530 Subject: [PATCH 063/277] Adding processQueryResponseResults2 (Independent pageFetcher and producer threads) --- .../google/cloud/bigquery/ConnectionImpl.java | 79 ++++++++++++++++++- 1 file changed, 78 insertions(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index bd0536e7a..23807739b 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -21,6 +21,7 @@ import com.google.api.services.bigquery.model.*; import com.google.cloud.RetryHelper; +import com.google.cloud.Tuple; import com.google.cloud.bigquery.spi.v2.BigQueryRpc; import com.google.common.base.Function; import com.google.common.base.Strings; @@ -162,7 +163,7 @@ public FieldValueList apply(TableRow rowPb) { } private BigQueryResultSet processQueryResponseResults( - com.google.api.services.bigquery.model.QueryResponse results) { + com.google.api.services.bigquery.model.QueryResponse results) { // Muttithreaded- Logic 1 Schema schema; long numRows; schema = Schema.fromPb(results.getSchema()); @@ -240,6 +241,82 @@ private BigQueryResultSet processQueryResponseResults( return new BigQueryResultSetImpl>(schema, numRows, buffer); } + private BigQueryResultSet processQueryResponseResults2( + com.google.api.services.bigquery.model.QueryResponse results) { // Muttithreaded- Logic 2 + Schema schema; + long numRows; + schema = Schema.fromPb(results.getSchema()); + numRows = results.getTotalRows().longValue(); + + // Producer thread for populating the buffer row by row + // Keeping the buffersize more than the page size and ensuring it's always a reasonable number + int bufferSize = + (int) + (connectionSettings.getNumBufferedRows() == null + || connectionSettings.getNumBufferedRows() < 10000 + ? 20000 + : (connectionSettings.getNumBufferedRows() * 2)); + BlockingQueue> buffer = new LinkedBlockingDeque<>(bufferSize); + BlockingQueue> nextPage = + new LinkedBlockingDeque<>(2); // we will be keeping atmost 2 page fetched upfront + + JobId jobId = JobId.fromPb(results.getJobReference()); + final TableId destinationTable = queryJobsGetRpc(jobId); + Runnable nextPageTask = + () -> { + String pageToken = results.getPageToken(); + try { + while (pageToken != null) { + TableDataList tabledataList = tableDataListRpc(destinationTable, pageToken); + nextPage.put(Tuple.of(tabledataList, true)); + pageToken = tabledataList.getPageToken(); + } + nextPage.put(Tuple.of(null, false)); // no further pages + } catch (InterruptedException e) { + e.printStackTrace(); // TODO: Throw an exception back + } + }; + Thread nextPageFetcher = new Thread(nextPageTask); + nextPageFetcher.start(); + + Runnable populateBufferRunnable = + () -> { // producer thread populating the buffer + Iterable fieldValueLists = + getIterableFieldValueList(results.getRows(), schema); + + boolean hasRows = true; // as we have to process the first page + + while (hasRows) { + for (FieldValueList fieldValueList : fieldValueLists) { + try { + buffer.put(fieldValueList); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + try { + Tuple nextPageTuple = nextPage.take(); + TableDataList tabledataList = nextPageTuple.x(); + hasRows = nextPageTuple.y(); + if (hasRows) { + fieldValueLists = getIterableFieldValueList(tabledataList.getRows(), schema); + } + } catch (InterruptedException e) { + e.printStackTrace(); // TODO: Throw an exception back + } + } + + buffer.offer( + new EndOfFieldValueList()); // All the pages has been processed, put this marker + }; + Thread populateBufferWorker = new Thread(populateBufferRunnable); + populateBufferWorker.start(); // producer process to populate the buffer + + // This will work for pagination as well, as buffer is getting updated asynchronously + return new BigQueryResultSetImpl>(schema, numRows, buffer); + } + // TODO(prasmish): Remove it after benchmarking @Deprecated private BigQueryResultSet processQueryResponseResultsSingleThreadedProducer( From d93d183b8bd5d6ff4a43c61cd93e093fae8bf2b7 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 10 Nov 2021 14:22:17 +0530 Subject: [PATCH 064/277] Adding processQueryResponseResults2 (Independent pageFetcher and producer threads) --- .../java/com/google/cloud/bigquery/ConnectionImpl.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 23807739b..fe0689e11 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -257,8 +257,10 @@ private BigQueryResultSet processQueryResponseResults2( ? 20000 : (connectionSettings.getNumBufferedRows() * 2)); BlockingQueue> buffer = new LinkedBlockingDeque<>(bufferSize); - BlockingQueue> nextPage = + BlockingQueue> pageCache = new LinkedBlockingDeque<>(2); // we will be keeping atmost 2 page fetched upfront + // TODO(prasmish) - Add a dynamic logic to set higher pageCache size if user selects a lower + // page size (as we can have more pages in cache) JobId jobId = JobId.fromPb(results.getJobReference()); final TableId destinationTable = queryJobsGetRpc(jobId); @@ -268,10 +270,10 @@ private BigQueryResultSet processQueryResponseResults2( try { while (pageToken != null) { TableDataList tabledataList = tableDataListRpc(destinationTable, pageToken); - nextPage.put(Tuple.of(tabledataList, true)); + pageCache.put(Tuple.of(tabledataList, true)); pageToken = tabledataList.getPageToken(); } - nextPage.put(Tuple.of(null, false)); // no further pages + pageCache.put(Tuple.of(null, false)); // no further pages } catch (InterruptedException e) { e.printStackTrace(); // TODO: Throw an exception back } @@ -296,7 +298,7 @@ private BigQueryResultSet processQueryResponseResults2( } try { - Tuple nextPageTuple = nextPage.take(); + Tuple nextPageTuple = pageCache.take(); TableDataList tabledataList = nextPageTuple.x(); hasRows = nextPageTuple.y(); if (hasRows) { From 6ba31b7a487188d9bcea1702e512502650279379 Mon Sep 17 00:00:00 2001 From: stephwang Date: Mon, 15 Nov 2021 15:39:04 -0500 Subject: [PATCH 065/277] nit: correct Javadoc --- .../java/com/google/cloud/bigquery/ConnectionSettings.java | 4 ++-- .../cloud/bigquery/ReadClientConnectionConfiguration.java | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java index 06349080e..e7dca72b8 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java @@ -253,8 +253,8 @@ public abstract static class Builder { /** * Sets the values necessary to determine whether table result will be read using the BigQuery * Storage client Read API. The BigQuery Storage client Read API will be used to read the query - * result when the totalToFirstPageSizeRatio (default 3) and minimumTableSize (default 100MB) - * conditions set are met. A ReadSession will be created using Apache Avro data format for + * result when the totalToFirstPageSizeRatio (default 3) and minimumTableSize (default 100 rows) + * conditions set are met. A ReadSession will be created using the Apache Arrow data format for * serialization. * *

It also sets the maximum number of table rows allowed in buffer before streaming them to diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ReadClientConnectionConfiguration.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ReadClientConnectionConfiguration.java index 96faa4802..483ec50e1 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ReadClientConnectionConfiguration.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ReadClientConnectionConfiguration.java @@ -41,7 +41,10 @@ public abstract static class Builder { @Nullable public abstract Builder setMinResultSize(Long numRows); - /** Sets the buffer size during streaming from the BigQueryStorage Read client. */ + /** + * Sets the maximum number of table rows allowed in buffer before streaming them to the + * BigQueryResultSet. + */ @Nullable public abstract Builder setBufferSize(Long bufferSize); From e6a480b9eb78da7fc71401c153c4f17d45beb073 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 16 Nov 2021 21:33:27 +0530 Subject: [PATCH 066/277] Added getPageCacheSize for dynamically computing the size of pageCache --- .../google/cloud/bigquery/ConnectionImpl.java | 37 ++++++++++++++++--- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index fe0689e11..0063c16ed 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -134,7 +134,7 @@ private BigQueryResultSet queryRpc(final String projectId, final QueryRequest qu // Query finished running and we can paginate all the results if (results.getJobComplete() && results.getSchema() != null) { - return processQueryResponseResults(results); + return processQueryResponseResults2(results); } else { // Query is long running (> 10s) and hasn't completed yet, or query completed but didn't // return the schema, fallback to jobs.insert path. Some operations don't return the schema @@ -176,7 +176,7 @@ private BigQueryResultSet processQueryResponseResults( (connectionSettings.getNumBufferedRows() == null || connectionSettings.getNumBufferedRows() < 10000 ? 20000 - : (connectionSettings.getNumBufferedRows() * 2)); + : (connectionSettings.getNumBufferedRows() * 2)); // TODO - fine tune the logic BlockingQueue> buffer = new LinkedBlockingDeque<>(bufferSize); BlockingQueue nextPage = new LinkedBlockingDeque<>(1); // we will be keeping atmost 1 page fetched upfront @@ -241,6 +241,34 @@ private BigQueryResultSet processQueryResponseResults( return new BigQueryResultSetImpl>(schema, numRows, buffer); } + // This methods tries to find the optimal number of page size depending of + private int getPageCacheSize(Long numBufferedRows, long numRows, Schema schema) { + final int MIN_CACHE_SIZE = 2; // Min number of pages in the page size + final int MAX_CACHE_SIZE = 20; // //Min number of pages in the page size + int columnsRead = schema.getFields().size(); + int pageCacheSize = 10; // default page size + // TODO: Further enhance this logic + if (numBufferedRows > 10000) { + pageCacheSize = + 2; // the size of numBufferedRows is quite large and as per our tests we should be able to + // do enough even with low + } + if (columnsRead > 15 + && numBufferedRows + > 5000) { // too many fields are being read, setting the page size on the lower end + pageCacheSize = 3; + } + if (numBufferedRows < 2000 + && columnsRead < 15) { // low pagesize with less number of columns, we can cache more pages + pageCacheSize = 20; + } + return pageCacheSize < MIN_CACHE_SIZE + ? MIN_CACHE_SIZE + : (Math.min( + pageCacheSize, + MAX_CACHE_SIZE)); // pageCacheSize should be between the defined min and max + } + private BigQueryResultSet processQueryResponseResults2( com.google.api.services.bigquery.model.QueryResponse results) { // Muttithreaded- Logic 2 Schema schema; @@ -258,9 +286,8 @@ private BigQueryResultSet processQueryResponseResults2( : (connectionSettings.getNumBufferedRows() * 2)); BlockingQueue> buffer = new LinkedBlockingDeque<>(bufferSize); BlockingQueue> pageCache = - new LinkedBlockingDeque<>(2); // we will be keeping atmost 2 page fetched upfront - // TODO(prasmish) - Add a dynamic logic to set higher pageCache size if user selects a lower - // page size (as we can have more pages in cache) + new LinkedBlockingDeque<>( + getPageCacheSize(connectionSettings.getNumBufferedRows(), numRows, schema)); JobId jobId = JobId.fromPb(results.getJobReference()); final TableId destinationTable = queryJobsGetRpc(jobId); From a4ec57f8ec04e29f30bacb66dca81072805da3b6 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 17 Nov 2021 16:01:08 +0530 Subject: [PATCH 067/277] Fixed NPE @ getPageCacheSize --- .../java/com/google/cloud/bigquery/ConnectionImpl.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 0063c16ed..7b6dab287 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -247,18 +247,20 @@ private int getPageCacheSize(Long numBufferedRows, long numRows, Schema schema) final int MAX_CACHE_SIZE = 20; // //Min number of pages in the page size int columnsRead = schema.getFields().size(); int pageCacheSize = 10; // default page size + long pageSize = numBufferedRows == null ? 0 : numBufferedRows.longValue(); + // TODO: Further enhance this logic - if (numBufferedRows > 10000) { + if (pageSize > 10000) { pageCacheSize = 2; // the size of numBufferedRows is quite large and as per our tests we should be able to // do enough even with low } if (columnsRead > 15 - && numBufferedRows + && pageSize > 5000) { // too many fields are being read, setting the page size on the lower end pageCacheSize = 3; } - if (numBufferedRows < 2000 + if (pageSize < 2000 && columnsRead < 15) { // low pagesize with less number of columns, we can cache more pages pageCacheSize = 20; } From c593545747a9252a4ca748f508a1c4b4ef362048 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 17 Nov 2021 18:36:28 +0530 Subject: [PATCH 068/277] Refactored - Renamed processQueryResponseResults methods --- .../google/cloud/bigquery/ConnectionImpl.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 7b6dab287..1f34263b8 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -134,7 +134,7 @@ private BigQueryResultSet queryRpc(final String projectId, final QueryRequest qu // Query finished running and we can paginate all the results if (results.getJobComplete() && results.getSchema() != null) { - return processQueryResponseResults2(results); + return processQueryResponseResults(results); } else { // Query is long running (> 10s) and hasn't completed yet, or query completed but didn't // return the schema, fallback to jobs.insert path. Some operations don't return the schema @@ -162,7 +162,9 @@ public FieldValueList apply(TableRow rowPb) { })); } - private BigQueryResultSet processQueryResponseResults( + // TODO(prasmish) - Delete this method after benchmarking + @Deprecated + private BigQueryResultSet processQueryResponseResults_Nested_NextPageTask( com.google.api.services.bigquery.model.QueryResponse results) { // Muttithreaded- Logic 1 Schema schema; long numRows; @@ -254,15 +256,15 @@ private int getPageCacheSize(Long numBufferedRows, long numRows, Schema schema) pageCacheSize = 2; // the size of numBufferedRows is quite large and as per our tests we should be able to // do enough even with low - } - if (columnsRead > 15 + } else if (columnsRead > 15 && pageSize > 5000) { // too many fields are being read, setting the page size on the lower end pageCacheSize = 3; - } - if (pageSize < 2000 + } else if (pageSize < 2000 && columnsRead < 15) { // low pagesize with less number of columns, we can cache more pages pageCacheSize = 20; + } else { // default - under 10K pageSize with any number of columns + pageCacheSize = 5; } return pageCacheSize < MIN_CACHE_SIZE ? MIN_CACHE_SIZE @@ -271,8 +273,9 @@ private int getPageCacheSize(Long numBufferedRows, long numRows, Schema schema) MAX_CACHE_SIZE)); // pageCacheSize should be between the defined min and max } - private BigQueryResultSet processQueryResponseResults2( - com.google.api.services.bigquery.model.QueryResponse results) { // Muttithreaded- Logic 2 + // this method uses two independent threads to query the pages and then populate the buffer + private BigQueryResultSet processQueryResponseResults( + com.google.api.services.bigquery.model.QueryResponse results) { Schema schema; long numRows; schema = Schema.fromPb(results.getSchema()); From dede418bd05b33ce37cff2a3d2d12fec5f012c1d Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 17 Nov 2021 19:09:36 +0530 Subject: [PATCH 069/277] Refactored - exception handling for processQueryResponseResults --- .../com/google/cloud/bigquery/ConnectionImpl.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 1f34263b8..342b04614 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -54,11 +54,14 @@ final class ConnectionImpl implements Connection { @Override public Boolean cancel() throws BigQuerySQLException { + // TODO: Stop the producer thread and the paging thread, clear the cache and buffer AFTER adding + // EoF there return null; } @Override public BigQueryDryRunResult dryRun(String sql) throws BigQuerySQLException { + // TODO: run a dummy query using tabaledata.list end point ? What are the query params return null; } @@ -307,7 +310,7 @@ private BigQueryResultSet processQueryResponseResults( } pageCache.put(Tuple.of(null, false)); // no further pages } catch (InterruptedException e) { - e.printStackTrace(); // TODO: Throw an exception back + throw new BigQueryException(0, e.getMessage(), e); } }; Thread nextPageFetcher = new Thread(nextPageTask); @@ -317,18 +320,15 @@ private BigQueryResultSet processQueryResponseResults( () -> { // producer thread populating the buffer Iterable fieldValueLists = getIterableFieldValueList(results.getRows(), schema); - boolean hasRows = true; // as we have to process the first page - while (hasRows) { for (FieldValueList fieldValueList : fieldValueLists) { try { buffer.put(fieldValueList); } catch (InterruptedException e) { - e.printStackTrace(); + throw new BigQueryException(0, e.getMessage(), e); } } - try { Tuple nextPageTuple = pageCache.take(); TableDataList tabledataList = nextPageTuple.x(); @@ -337,10 +337,9 @@ private BigQueryResultSet processQueryResponseResults( fieldValueLists = getIterableFieldValueList(tabledataList.getRows(), schema); } } catch (InterruptedException e) { - e.printStackTrace(); // TODO: Throw an exception back + throw new BigQueryException(0, e.getMessage(), e); } } - buffer.offer( new EndOfFieldValueList()); // All the pages has been processed, put this marker }; From 6c79796ed7dbbdee795bdca6bfeae873908c2e0f Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 19 Nov 2021 16:40:45 +0530 Subject: [PATCH 070/277] Added testConnectionCancel for testing ConnectionImpl.cancel --- .../cloud/bigquery/it/ITBigQueryTest.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 56f8b5ec6..b9df3a6d0 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -2280,6 +2280,34 @@ public void testExecuteSelectSinglePageTableRow() throws SQLException { assertFalse(rs.next()); // no 3rd row in the table } + @Test + public void testConnectionCancel() throws SQLException { + String query = + "SELECT date, county, state_name, confirmed_cases, deaths FROM " + + TABLE_ID_LARGE.getTable() + + " where date is not null and county is not null and state_name is not null order by date limit 300000"; + ConnectionSettings connectionSettings = + ConnectionSettings.newBuilder() + .setDefaultDataset(DatasetId.of(DATASET)) + .setNumBufferedRows(10000L) // page size + .build(); + Connection connection = bigquery.createConnection(connectionSettings); + BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); + ResultSet rs = bigQueryResultSet.getResultSet(); + int cnt = 0; + while (rs.next()) { + ++cnt; + if (cnt > 57000) {//breaking at 57K, query reads 300K + assertTrue(connection.cancel()); // we should be able to cancel the connection + } + } + assertTrue( + cnt + < 60000); // Few extra records are still read (generally ~10) even after canceling, as + // the backgrounds threads are still active while the interrupt occurs and the + // buffer and pageCache are cleared + } + @Test public void testBQResultSetPagination() throws SQLException { String query = From 4156217f31b31482d6fb3640fd4d33241156dfe0 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 19 Nov 2021 16:44:39 +0530 Subject: [PATCH 071/277] Added ConnectionImpl.cancel implementation and the interrupt logic in processQueryResponseResults --- .../google/cloud/bigquery/ConnectionImpl.java | 61 ++++++++++++++++--- 1 file changed, 51 insertions(+), 10 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 342b04614..36695758a 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -40,6 +40,11 @@ final class ConnectionImpl implements Connection { private BigQueryOptions bigQueryOptions; private BigQueryRpc bigQueryRpc; private BigQueryRetryConfig retryConfig; + private Thread pageFetcher, producerWorker; + private volatile boolean isCancelled = + false; // TODO: this flag is kind of redundant, but this.pageFetcher.isInterrupted() returns + // false even after interrupting it here! This could be due to the retrial logic, so + // using it as a workaround. ConnectionImpl( ConnectionSettings connectionSettings, @@ -52,16 +57,33 @@ final class ConnectionImpl implements Connection { this.retryConfig = retryConfig; } + /* + Cancel method shutdowns the pageFetcher and producerWorker threads gracefully using interrupt + + The pageFetcher threat wont request for any subsequent threads after interrupting and shutdown as soon as any ongoing RPC call returns + + The producerWorker wont populate the buffer with any further records and clear the buffer, put a EoF marker and shutdown + */ @Override - public Boolean cancel() throws BigQuerySQLException { - // TODO: Stop the producer thread and the paging thread, clear the cache and buffer AFTER adding - // EoF there - return null; + public synchronized Boolean cancel() throws BigQuerySQLException { + + // Interrupt the pageFetcher thread + if (this.pageFetcher != null && !this.pageFetcher.isInterrupted()) { + this.pageFetcher.interrupt(); + } + + // Interrupt the producerWorker thread + if (this.producerWorker != null && !this.producerWorker.isInterrupted()) { + this.producerWorker.interrupt(); + } + isCancelled = true; + return this.pageFetcher.isInterrupted() && this.producerWorker.isInterrupted(); } @Override public BigQueryDryRunResult dryRun(String sql) throws BigQuerySQLException { // TODO: run a dummy query using tabaledata.list end point ? What are the query params + // jobs.query or jobs.getqueuryresults queryrequestinfo.java return null; } @@ -251,8 +273,8 @@ private int getPageCacheSize(Long numBufferedRows, long numRows, Schema schema) final int MIN_CACHE_SIZE = 2; // Min number of pages in the page size final int MAX_CACHE_SIZE = 20; // //Min number of pages in the page size int columnsRead = schema.getFields().size(); - int pageCacheSize = 10; // default page size - long pageSize = numBufferedRows == null ? 0 : numBufferedRows.longValue(); + int pageCacheSize = 10; // default page size//TODO: numCachedPages + long pageSize = numBufferedRows == null ? 0 : numBufferedRows.longValue(); // TODO: Rename it // TODO: Further enhance this logic if (pageSize > 10000) { @@ -304,6 +326,10 @@ private BigQueryResultSet processQueryResponseResults( String pageToken = results.getPageToken(); try { while (pageToken != null) { + if (Thread.currentThread().isInterrupted() + || isCancelled) { // do not process further pages and shutdown + break; + } TableDataList tabledataList = tableDataListRpc(destinationTable, pageToken); pageCache.put(Tuple.of(tabledataList, true)); pageToken = tabledataList.getPageToken(); @@ -313,8 +339,8 @@ private BigQueryResultSet processQueryResponseResults( throw new BigQueryException(0, e.getMessage(), e); } }; - Thread nextPageFetcher = new Thread(nextPageTask); - nextPageFetcher.start(); + this.pageFetcher = new Thread(nextPageTask); + this.pageFetcher.start(); Runnable populateBufferRunnable = () -> { // producer thread populating the buffer @@ -324,11 +350,19 @@ private BigQueryResultSet processQueryResponseResults( while (hasRows) { for (FieldValueList fieldValueList : fieldValueLists) { try { + if (Thread.currentThread() + .isInterrupted()) { // do not process further pages and shutdown (inner loop) + break; + } buffer.put(fieldValueList); } catch (InterruptedException e) { throw new BigQueryException(0, e.getMessage(), e); } } + if (Thread.currentThread() + .isInterrupted()) { // do not process further pages and shutdown (outerloop) + break; + } try { Tuple nextPageTuple = pageCache.take(); TableDataList tabledataList = nextPageTuple.x(); @@ -340,11 +374,18 @@ private BigQueryResultSet processQueryResponseResults( throw new BigQueryException(0, e.getMessage(), e); } } + if (Thread.currentThread() + .isInterrupted()) { // clear the buffer for any outstanding records + buffer.clear(); + pageCache + .clear(); // IMP - so that if it's full then it unblocks and the interrupt logic + // could trigger + } buffer.offer( new EndOfFieldValueList()); // All the pages has been processed, put this marker }; - Thread populateBufferWorker = new Thread(populateBufferRunnable); - populateBufferWorker.start(); // producer process to populate the buffer + this.producerWorker = new Thread(populateBufferRunnable); + this.producerWorker.start(); // producer process to populate the buffer // This will work for pagination as well, as buffer is getting updated asynchronously return new BigQueryResultSetImpl>(schema, numRows, buffer); From 07885e6bd779fbc4f3cd4e52ef6e9b66b0606a3b Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 19 Nov 2021 16:49:38 +0530 Subject: [PATCH 072/277] Lint --- .../com/google/cloud/bigquery/it/ITBigQueryTest.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index b9df3a6d0..58cf1373d 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -2297,15 +2297,14 @@ public void testConnectionCancel() throws SQLException { int cnt = 0; while (rs.next()) { ++cnt; - if (cnt > 57000) {//breaking at 57K, query reads 300K + if (cnt > 57000) { // breaking at 57K, query reads 300K assertTrue(connection.cancel()); // we should be able to cancel the connection } } assertTrue( - cnt - < 60000); // Few extra records are still read (generally ~10) even after canceling, as - // the backgrounds threads are still active while the interrupt occurs and the - // buffer and pageCache are cleared + cnt < 60000); // Few extra records are still read (generally ~10) even after canceling, as + // the backgrounds threads are still active while the interrupt occurs and the + // buffer and pageCache are cleared } @Test From 109d1235f0d4e6a839bc5f07e3111865f723998c Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Mon, 22 Nov 2021 16:38:41 +0530 Subject: [PATCH 073/277] Commented testConnectionCancel - Need to update the Cancel Method logic --- .../java/com/google/cloud/bigquery/it/ITBigQueryTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 58cf1373d..a88fa8306 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -2279,7 +2279,7 @@ public void testExecuteSelectSinglePageTableRow() throws SQLException { assertFalse(rs.next()); // no 3rd row in the table } - + /* @Test public void testConnectionCancel() throws SQLException { String query = @@ -2305,7 +2305,7 @@ public void testConnectionCancel() throws SQLException { cnt < 60000); // Few extra records are still read (generally ~10) even after canceling, as // the backgrounds threads are still active while the interrupt occurs and the // buffer and pageCache are cleared - } + }*/ @Test public void testBQResultSetPagination() throws SQLException { From 1a582f36e0a2394dc23ca651db0dcef2b349600d Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Mon, 22 Nov 2021 16:39:36 +0530 Subject: [PATCH 074/277] Implemented processQueryResponseResults_thread_pooled --- .../google/cloud/bigquery/ConnectionImpl.java | 148 +++++++++++++++++- 1 file changed, 147 insertions(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 36695758a..0019157a7 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -30,6 +30,8 @@ import com.google.common.collect.Maps; import java.util.*; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingDeque; import java.util.stream.Collectors; @@ -45,6 +47,9 @@ final class ConnectionImpl implements Connection { false; // TODO: this flag is kind of redundant, but this.pageFetcher.isInterrupted() returns // false even after interrupting it here! This could be due to the retrial logic, so // using it as a workaround. + private final int MAX_PROCESS_QUERY_THREADS_CNT = 5; + private ExecutorService queryTaskExecutor = + Executors.newFixedThreadPool(MAX_PROCESS_QUERY_THREADS_CNT); ConnectionImpl( ConnectionSettings connectionSettings, @@ -159,7 +164,8 @@ private BigQueryResultSet queryRpc(final String projectId, final QueryRequest qu // Query finished running and we can paginate all the results if (results.getJobComplete() && results.getSchema() != null) { - return processQueryResponseResults(results); + return processQueryResponseResults_thread_pooled( + results); // processQueryResponseResults(results); } else { // Query is long running (> 10s) and hasn't completed yet, or query completed but didn't // return the schema, fallback to jobs.insert path. Some operations don't return the schema @@ -391,6 +397,146 @@ private BigQueryResultSet processQueryResponseResults( return new BigQueryResultSetImpl>(schema, numRows, buffer); } + private BigQueryResultSet processQueryResponseResults_thread_pooled( + com.google.api.services.bigquery.model.QueryResponse results) { + Schema schema; + long numRows; + schema = Schema.fromPb(results.getSchema()); + numRows = results.getTotalRows().longValue(); + + // Producer thread for populating the buffer row by row + // Keeping the buffersize more than the page size and ensuring it's always a reasonable number + int bufferSize = + (int) + (connectionSettings.getNumBufferedRows() == null + || connectionSettings.getNumBufferedRows() < 10000 + ? 20000 + : (connectionSettings.getNumBufferedRows() * 2)); + BlockingQueue> buffer = new LinkedBlockingDeque<>(bufferSize); + BlockingQueue, Boolean>> pageCache = + new LinkedBlockingDeque<>( + getPageCacheSize(connectionSettings.getNumBufferedRows(), numRows, schema)); + + JobId jobId = JobId.fromPb(results.getJobReference()); + final TableId destinationTable = queryJobsGetRpc(jobId); + + Runnable nextPageTask = + () -> { + // long totalTime = 0, curTime =0; + String pageToken = results.getPageToken(); + try { + while (pageToken != null) { + + if (Thread.currentThread().isInterrupted() + || isCancelled) { // do not process further pages and shutdown + break; + } + TableDataList tabledataList = tableDataListRpc(destinationTable, pageToken); + pageToken = tabledataList.getPageToken(); + parseDataAsync( + tabledataList, + schema, + pageCache, + pageToken); // parses data on a separate thread, thus maximising processing s + // throughput + // TODO: Make sure that the order is maintained (Req in corner case when the parsing + // gets slower than the network I/O) + + /* // curTime = System.currentTimeMillis(); + Iterable fieldValueLists = + getIterableFieldValueList(tabledataList.getRows(), schema); // Parse + // totalTime += (System.currentTimeMillis()-curTime); + pageCache.put(Tuple.of(fieldValueLists, true));*/ + + } + // System.out.println("Parsing time: "+totalTime); + // pageCache.put(Tuple.of(null, false)); // no further pages + } catch (Exception e) { + throw new BigQueryException(0, e.getMessage(), e); + } + }; + queryTaskExecutor.execute(nextPageTask); + + populateBuffer(results, schema, pageCache, buffer); // spawns a thread to populate the buffer + + // This will work for pagination as well, as buffer is getting updated asynchronously + return new BigQueryResultSetImpl>(schema, numRows, buffer); + } + + private void parseDataAsync( + TableDataList tabledataList, + Schema schema, + BlockingQueue, Boolean>> pageCache, + String pageToken) { + Runnable parseDataTask = + () -> { + try { + Iterable fieldValueLists = + getIterableFieldValueList(tabledataList.getRows(), schema); // Parse + + pageCache.put(Tuple.of(fieldValueLists, true)); + + if (pageToken == null) { + pageCache.put(Tuple.of(null, false)); // no further pages + } + } catch (InterruptedException e) { + throw new BigQueryException(0, e.getMessage(), e); + } + }; + queryTaskExecutor.execute(parseDataTask); + } + + private void populateBuffer( + com.google.api.services.bigquery.model.QueryResponse results, + Schema schema, + BlockingQueue, Boolean>> pageCache, + BlockingQueue> buffer) { + Runnable populateBufferRunnable = + () -> { // producer thread populating the buffer + Iterable fieldValueLists = + getIterableFieldValueList(results.getRows(), schema); + boolean hasRows = true; // as we have to process the first page + while (hasRows) { + for (FieldValueList fieldValueList : fieldValueLists) { + try { + if (Thread.currentThread() + .isInterrupted()) { // do not process further pages and shutdown (inner loop) + break; + } + buffer.put(fieldValueList); + } catch (InterruptedException e) { + throw new BigQueryException(0, e.getMessage(), e); + } + } + if (Thread.currentThread() + .isInterrupted()) { // do not process further pages and shutdown (outerloop) + break; + } + try { + Tuple, Boolean> nextPageTuple = pageCache.take(); + hasRows = nextPageTuple.y(); + if (hasRows) { + fieldValueLists = nextPageTuple.x(); + } + } catch (InterruptedException e) { + throw new BigQueryException(0, e.getMessage(), e); + } + } + if (Thread.currentThread() + .isInterrupted()) { // clear the buffer for any outstanding records + buffer.clear(); + pageCache + .clear(); // IMP - so that if it's full then it unblocks and the interrupt logic + // could trigger + } + buffer.offer( + new EndOfFieldValueList()); // All the pages has been processed, put this marker + queryTaskExecutor.shutdownNow(); + }; + + queryTaskExecutor.execute(populateBufferRunnable); + } + // TODO(prasmish): Remove it after benchmarking @Deprecated private BigQueryResultSet processQueryResponseResultsSingleThreadedProducer( From b8e39fae085e7028d3b94a0a8034f0702adbdd98 Mon Sep 17 00:00:00 2001 From: stephwang Date: Mon, 22 Nov 2021 12:02:00 -0500 Subject: [PATCH 075/277] nit: rename a few vars --- .../google/cloud/bigquery/ConnectionImpl.java | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 0019157a7..0f3ed5edf 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -274,34 +274,34 @@ private BigQueryResultSet processQueryResponseResults_Nested_NextPageTask( return new BigQueryResultSetImpl>(schema, numRows, buffer); } - // This methods tries to find the optimal number of page size depending of + // Determines the optimal number of caches pages to improve read performance private int getPageCacheSize(Long numBufferedRows, long numRows, Schema schema) { final int MIN_CACHE_SIZE = 2; // Min number of pages in the page size final int MAX_CACHE_SIZE = 20; // //Min number of pages in the page size int columnsRead = schema.getFields().size(); - int pageCacheSize = 10; // default page size//TODO: numCachedPages - long pageSize = numBufferedRows == null ? 0 : numBufferedRows.longValue(); // TODO: Rename it + int numCachedPages = 10; // default page size + long numCachedRows = numBufferedRows == null ? 0 : numBufferedRows.longValue(); // TODO: Further enhance this logic - if (pageSize > 10000) { - pageCacheSize = + if (numCachedRows > 10000) { + numCachedPages = 2; // the size of numBufferedRows is quite large and as per our tests we should be able to // do enough even with low } else if (columnsRead > 15 - && pageSize + && numCachedRows > 5000) { // too many fields are being read, setting the page size on the lower end - pageCacheSize = 3; - } else if (pageSize < 2000 - && columnsRead < 15) { // low pagesize with less number of columns, we can cache more pages - pageCacheSize = 20; - } else { // default - under 10K pageSize with any number of columns - pageCacheSize = 5; - } - return pageCacheSize < MIN_CACHE_SIZE + numCachedPages = 3; + } else if (numCachedRows < 2000 + && columnsRead < 15) { // low pagesize with fewer number of columns, we can cache more pages + numCachedPages = 20; + } else { // default - under 10K numCachedRows with any number of columns + numCachedPages = 5; + } + return numCachedPages < MIN_CACHE_SIZE ? MIN_CACHE_SIZE : (Math.min( - pageCacheSize, - MAX_CACHE_SIZE)); // pageCacheSize should be between the defined min and max + numCachedPages, + MAX_CACHE_SIZE)); // numCachedPages should be between the defined min and max } // this method uses two independent threads to query the pages and then populate the buffer From 7bd4e7b1d4ee85c8b150d2b5f96645e64e99338e Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 23 Nov 2021 13:52:33 +0530 Subject: [PATCH 076/277] Uncommented testConnectionCancel --- .../java/com/google/cloud/bigquery/it/ITBigQueryTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index a88fa8306..58cf1373d 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -2279,7 +2279,7 @@ public void testExecuteSelectSinglePageTableRow() throws SQLException { assertFalse(rs.next()); // no 3rd row in the table } - /* + @Test public void testConnectionCancel() throws SQLException { String query = @@ -2305,7 +2305,7 @@ public void testConnectionCancel() throws SQLException { cnt < 60000); // Few extra records are still read (generally ~10) even after canceling, as // the backgrounds threads are still active while the interrupt occurs and the // buffer and pageCache are cleared - }*/ + } @Test public void testBQResultSetPagination() throws SQLException { From 6748795d01132c17ae86a32c8f8425a66884ff7d Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 23 Nov 2021 13:53:49 +0530 Subject: [PATCH 077/277] Cleaned processQueryResponseResults. Removed unused methods. Implemented ThreadPool --- .../google/cloud/bigquery/ConnectionImpl.java | 413 +++++------------- 1 file changed, 111 insertions(+), 302 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 0f3ed5edf..4c85d0742 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -33,6 +33,8 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingDeque; +import java.util.logging.Level; +import java.util.logging.Logger; import java.util.stream.Collectors; /** Implementation for {@link Connection}, the generic BigQuery connection API (not JDBC). */ @@ -42,14 +44,12 @@ final class ConnectionImpl implements Connection { private BigQueryOptions bigQueryOptions; private BigQueryRpc bigQueryRpc; private BigQueryRetryConfig retryConfig; - private Thread pageFetcher, producerWorker; - private volatile boolean isCancelled = - false; // TODO: this flag is kind of redundant, but this.pageFetcher.isInterrupted() returns // false even after interrupting it here! This could be due to the retrial logic, so // using it as a workaround. private final int MAX_PROCESS_QUERY_THREADS_CNT = 5; private ExecutorService queryTaskExecutor = Executors.newFixedThreadPool(MAX_PROCESS_QUERY_THREADS_CNT); + private final Logger logger = Logger.getLogger(this.getClass().getName()); ConnectionImpl( ConnectionSettings connectionSettings, @@ -71,18 +71,8 @@ final class ConnectionImpl implements Connection { */ @Override public synchronized Boolean cancel() throws BigQuerySQLException { - - // Interrupt the pageFetcher thread - if (this.pageFetcher != null && !this.pageFetcher.isInterrupted()) { - this.pageFetcher.interrupt(); - } - - // Interrupt the producerWorker thread - if (this.producerWorker != null && !this.producerWorker.isInterrupted()) { - this.producerWorker.interrupt(); - } - isCancelled = true; - return this.pageFetcher.isInterrupted() && this.producerWorker.isInterrupted(); + queryTaskExecutor.shutdownNow(); + return queryTaskExecutor.isShutdown(); } @Override @@ -164,8 +154,7 @@ private BigQueryResultSet queryRpc(final String projectId, final QueryRequest qu // Query finished running and we can paginate all the results if (results.getJobComplete() && results.getSchema() != null) { - return processQueryResponseResults_thread_pooled( - results); // processQueryResponseResults(results); + return processQueryResponseResults(results); // processQueryResponseResults(results); } else { // Query is long running (> 10s) and hasn't completed yet, or query completed but didn't // return the schema, fallback to jobs.insert path. Some operations don't return the schema @@ -193,87 +182,6 @@ public FieldValueList apply(TableRow rowPb) { })); } - // TODO(prasmish) - Delete this method after benchmarking - @Deprecated - private BigQueryResultSet processQueryResponseResults_Nested_NextPageTask( - com.google.api.services.bigquery.model.QueryResponse results) { // Muttithreaded- Logic 1 - Schema schema; - long numRows; - schema = Schema.fromPb(results.getSchema()); - numRows = results.getTotalRows().longValue(); - - // Producer thread for populating the buffer row by row - // Keeping the buffersize more than the page size and ensuring it's always a reasonable number - int bufferSize = - (int) - (connectionSettings.getNumBufferedRows() == null - || connectionSettings.getNumBufferedRows() < 10000 - ? 20000 - : (connectionSettings.getNumBufferedRows() * 2)); // TODO - fine tune the logic - BlockingQueue> buffer = new LinkedBlockingDeque<>(bufferSize); - BlockingQueue nextPage = - new LinkedBlockingDeque<>(1); // we will be keeping atmost 1 page fetched upfront - - Runnable populateBufferRunnable = - () -> { // producer thread populating the buffer - Iterable fieldValueLists = - getIterableFieldValueList(results.getRows(), schema); - - boolean hasRows = true; // as we have to process the first page - String pageToken = results.getPageToken(); - JobId jobId = JobId.fromPb(results.getJobReference()); - final TableId destinationTable = queryJobsGetRpc(jobId); - while (hasRows) { - if (pageToken - != null) { // spawn a child thread to fetch the next page before the current page is - // processed - final String nextPageToken = pageToken; - Runnable nextPageTask = - () -> { - try { - TableDataList tabledataList = - tableDataListRpc(destinationTable, nextPageToken); - nextPage.put(tabledataList); - } catch (InterruptedException e) { - e.printStackTrace(); // TODO: Throw an exception back - } - }; - Thread nextPageFetcher = new Thread(nextPageTask); - nextPageFetcher.start(); - } - - for (FieldValueList fieldValueList : fieldValueLists) { - try { - buffer.put(fieldValueList); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - if (pageToken != null) { - // next page would have been requested using a child process, check if the it's - // available - try { - TableDataList tabledataList = nextPage.take(); - fieldValueLists = getIterableFieldValueList(tabledataList.getRows(), schema); - hasRows = true; // new page was requested, which is not yet processed - pageToken = tabledataList.getPageToken(); // next page's token - } catch (InterruptedException e) { - e.printStackTrace(); // TODO: Throw an exception back - } - } else { // the current page has been processed and there's no next page - hasRows = false; - } - } - buffer.offer( - new EndOfFieldValueList()); // All the pages has been processed, put this marker - }; - Thread populateBufferWorker = new Thread(populateBufferRunnable); - populateBufferWorker.start(); // producer process to populate the buffer - - // This will work for pagination as well, as buffer is getting updated asynchronously - return new BigQueryResultSetImpl>(schema, numRows, buffer); - } - // Determines the optimal number of caches pages to improve read performance private int getPageCacheSize(Long numBufferedRows, long numRows, Schema schema) { final int MIN_CACHE_SIZE = 2; // Min number of pages in the page size @@ -304,7 +212,6 @@ private int getPageCacheSize(Long numBufferedRows, long numRows, Schema schema) MAX_CACHE_SIZE)); // numCachedPages should be between the defined min and max } - // this method uses two independent threads to query the pages and then populate the buffer private BigQueryResultSet processQueryResponseResults( com.google.api.services.bigquery.model.QueryResponse results) { Schema schema; @@ -320,183 +227,148 @@ private BigQueryResultSet processQueryResponseResults( || connectionSettings.getNumBufferedRows() < 10000 ? 20000 : (connectionSettings.getNumBufferedRows() * 2)); - BlockingQueue> buffer = new LinkedBlockingDeque<>(bufferSize); - BlockingQueue> pageCache = + BlockingQueue> buffer = new LinkedBlockingDeque<>( - getPageCacheSize(connectionSettings.getNumBufferedRows(), numRows, schema)); - - JobId jobId = JobId.fromPb(results.getJobReference()); - final TableId destinationTable = queryJobsGetRpc(jobId); - Runnable nextPageTask = - () -> { - String pageToken = results.getPageToken(); - try { - while (pageToken != null) { - if (Thread.currentThread().isInterrupted() - || isCancelled) { // do not process further pages and shutdown - break; - } - TableDataList tabledataList = tableDataListRpc(destinationTable, pageToken); - pageCache.put(Tuple.of(tabledataList, true)); - pageToken = tabledataList.getPageToken(); - } - pageCache.put(Tuple.of(null, false)); // no further pages - } catch (InterruptedException e) { - throw new BigQueryException(0, e.getMessage(), e); - } - }; - this.pageFetcher = new Thread(nextPageTask); - this.pageFetcher.start(); - - Runnable populateBufferRunnable = - () -> { // producer thread populating the buffer - Iterable fieldValueLists = - getIterableFieldValueList(results.getRows(), schema); - boolean hasRows = true; // as we have to process the first page - while (hasRows) { - for (FieldValueList fieldValueList : fieldValueLists) { - try { - if (Thread.currentThread() - .isInterrupted()) { // do not process further pages and shutdown (inner loop) - break; - } - buffer.put(fieldValueList); - } catch (InterruptedException e) { - throw new BigQueryException(0, e.getMessage(), e); - } - } - if (Thread.currentThread() - .isInterrupted()) { // do not process further pages and shutdown (outerloop) - break; - } - try { - Tuple nextPageTuple = pageCache.take(); - TableDataList tabledataList = nextPageTuple.x(); - hasRows = nextPageTuple.y(); - if (hasRows) { - fieldValueLists = getIterableFieldValueList(tabledataList.getRows(), schema); - } - } catch (InterruptedException e) { - throw new BigQueryException(0, e.getMessage(), e); - } - } - if (Thread.currentThread() - .isInterrupted()) { // clear the buffer for any outstanding records - buffer.clear(); - pageCache - .clear(); // IMP - so that if it's full then it unblocks and the interrupt logic - // could trigger - } - buffer.offer( - new EndOfFieldValueList()); // All the pages has been processed, put this marker - }; - this.producerWorker = new Thread(populateBufferRunnable); - this.producerWorker.start(); // producer process to populate the buffer - - // This will work for pagination as well, as buffer is getting updated asynchronously - return new BigQueryResultSetImpl>(schema, numRows, buffer); - } - - private BigQueryResultSet processQueryResponseResults_thread_pooled( - com.google.api.services.bigquery.model.QueryResponse results) { - Schema schema; - long numRows; - schema = Schema.fromPb(results.getSchema()); - numRows = results.getTotalRows().longValue(); - - // Producer thread for populating the buffer row by row - // Keeping the buffersize more than the page size and ensuring it's always a reasonable number - int bufferSize = - (int) - (connectionSettings.getNumBufferedRows() == null - || connectionSettings.getNumBufferedRows() < 10000 - ? 20000 - : (connectionSettings.getNumBufferedRows() * 2)); - BlockingQueue> buffer = new LinkedBlockingDeque<>(bufferSize); + bufferSize); // this keeps the deserialized records at the row level, which will be + // consumed by the BQResultSet BlockingQueue, Boolean>> pageCache = new LinkedBlockingDeque<>( - getPageCacheSize(connectionSettings.getNumBufferedRows(), numRows, schema)); + getPageCacheSize( + connectionSettings.getNumBufferedRows(), + numRows, + schema)); // this keeps the parsed FieldValueLists + BlockingQueue> rpcResponseQueue = + new LinkedBlockingDeque<>( + getPageCacheSize( + connectionSettings.getNumBufferedRows(), + numRows, + schema)); // this keeps the raw RPC response JobId jobId = JobId.fromPb(results.getJobReference()); final TableId destinationTable = queryJobsGetRpc(jobId); + // This thread makes the RPC calls and paginates Runnable nextPageTask = () -> { - // long totalTime = 0, curTime =0; String pageToken = results.getPageToken(); try { - while (pageToken != null) { - + while (pageToken != null) { // paginate for non null token if (Thread.currentThread().isInterrupted() - || isCancelled) { // do not process further pages and shutdown + || queryTaskExecutor.isShutdown()) { // do not process further pages and shutdown break; } TableDataList tabledataList = tableDataListRpc(destinationTable, pageToken); pageToken = tabledataList.getPageToken(); - parseDataAsync( - tabledataList, - schema, - pageCache, - pageToken); // parses data on a separate thread, thus maximising processing s - // throughput - // TODO: Make sure that the order is maintained (Req in corner case when the parsing - // gets slower than the network I/O) - - /* // curTime = System.currentTimeMillis(); - Iterable fieldValueLists = - getIterableFieldValueList(tabledataList.getRows(), schema); // Parse - // totalTime += (System.currentTimeMillis()-curTime); - pageCache.put(Tuple.of(fieldValueLists, true));*/ - + rpcResponseQueue.put( + Tuple.of( + tabledataList, + true)); // this will be parsed asynchronously without blocking the current + // thread } - // System.out.println("Parsing time: "+totalTime); - // pageCache.put(Tuple.of(null, false)); // no further pages + rpcResponseQueue.put( + Tuple.of( + null, + false)); // this will stop the parseDataTask as well in case of interrupt or + // when the pagination completes } catch (Exception e) { throw new BigQueryException(0, e.getMessage(), e); } }; queryTaskExecutor.execute(nextPageTask); - populateBuffer(results, schema, pageCache, buffer); // spawns a thread to populate the buffer + parseRpcDataAsync( + results, + schema, + pageCache, + rpcResponseQueue); // parses data on a separate thread, thus maximising processing + // throughput + + populateBufferAsync( + rpcResponseQueue, pageCache, buffer); // spawns a thread to populate the buffer // This will work for pagination as well, as buffer is getting updated asynchronously return new BigQueryResultSetImpl>(schema, numRows, buffer); } - private void parseDataAsync( - TableDataList tabledataList, + /* + This method takes TableDataList from rpcResponseQueue and populates pageCache with FieldValueList + */ + private void parseRpcDataAsync( + com.google.api.services.bigquery.model.QueryResponse results, Schema schema, BlockingQueue, Boolean>> pageCache, - String pageToken) { + BlockingQueue> rpcResponseQueue) { + + // parse and put the first page in the pageCache before the other pages are parsed from the RPC + // calls + Iterable firstFieldValueLists = + getIterableFieldValueList(results.getRows(), schema); + try { + pageCache.put( + Tuple.of(firstFieldValueLists, true)); // this is the first page which we have received. + } catch (InterruptedException e) { + throw new BigQueryException(0, e.getMessage(), e); + } + + // rpcResponseQueue will get null tuple if Cancel method is called, so no need to explicitly use + // thread interrupt here Runnable parseDataTask = () -> { try { - Iterable fieldValueLists = - getIterableFieldValueList(tabledataList.getRows(), schema); // Parse - - pageCache.put(Tuple.of(fieldValueLists, true)); - - if (pageToken == null) { - pageCache.put(Tuple.of(null, false)); // no further pages + boolean hasMorePages = true; + while (hasMorePages) { + Tuple rpcResponse = rpcResponseQueue.take(); + TableDataList tabledataList = rpcResponse.x(); + hasMorePages = rpcResponse.y(); + if (tabledataList != null) { + Iterable fieldValueLists = + getIterableFieldValueList(tabledataList.getRows(), schema); // Parse + pageCache.put(Tuple.of(fieldValueLists, true)); + } } } catch (InterruptedException e) { - throw new BigQueryException(0, e.getMessage(), e); + logger.log( + Level.WARNING, + "\n" + Thread.currentThread().getName() + " Interrupted", + e); // Thread might get interrupted while calling the Cancel method, which is + // expected, so logging this instead of throwing the exception back + } + try { + pageCache.put(Tuple.of(null, false)); // no further pages + } catch (InterruptedException e) { + e.printStackTrace(); } }; queryTaskExecutor.execute(parseDataTask); } - private void populateBuffer( - com.google.api.services.bigquery.model.QueryResponse results, - Schema schema, + private void populateBufferAsync( + BlockingQueue> rpcResponseQueue, BlockingQueue, Boolean>> pageCache, BlockingQueue> buffer) { Runnable populateBufferRunnable = () -> { // producer thread populating the buffer - Iterable fieldValueLists = - getIterableFieldValueList(results.getRows(), schema); + Iterable fieldValueLists = null; boolean hasRows = true; // as we have to process the first page while (hasRows) { + try { + Tuple, Boolean> nextPageTuple = pageCache.take(); + hasRows = nextPageTuple.y(); + fieldValueLists = nextPageTuple.x(); + } catch (InterruptedException e) { + logger.log( + Level.WARNING, + "\n" + Thread.currentThread().getName() + " Interrupted", + e); // Thread might get interrupted while calling the Cancel method, which is + // expected, so logging this instead of throwing the exception back + } + + if (Thread.currentThread().isInterrupted() + || fieldValueLists + == null) { // do not process further pages and shutdown (outerloop) + break; + } + for (FieldValueList fieldValueList : fieldValueLists) { try { if (Thread.currentThread() @@ -508,89 +380,26 @@ private void populateBuffer( throw new BigQueryException(0, e.getMessage(), e); } } - if (Thread.currentThread() - .isInterrupted()) { // do not process further pages and shutdown (outerloop) - break; - } - try { - Tuple, Boolean> nextPageTuple = pageCache.take(); - hasRows = nextPageTuple.y(); - if (hasRows) { - fieldValueLists = nextPageTuple.x(); - } - } catch (InterruptedException e) { - throw new BigQueryException(0, e.getMessage(), e); - } } + if (Thread.currentThread() .isInterrupted()) { // clear the buffer for any outstanding records buffer.clear(); - pageCache + rpcResponseQueue .clear(); // IMP - so that if it's full then it unblocks and the interrupt logic // could trigger } - buffer.offer( - new EndOfFieldValueList()); // All the pages has been processed, put this marker - queryTaskExecutor.shutdownNow(); - }; - - queryTaskExecutor.execute(populateBufferRunnable); - } - // TODO(prasmish): Remove it after benchmarking - @Deprecated - private BigQueryResultSet processQueryResponseResultsSingleThreadedProducer( - com.google.api.services.bigquery.model.QueryResponse results) { - Schema schema; - long numRows; - schema = Schema.fromPb(results.getSchema()); - numRows = results.getTotalRows().longValue(); - - // Producer thread for populating the buffer row by row - // Keeping the buffersize more than the page size and ensuring it's always a reasonable number - int bufferSize = - (int) - (connectionSettings.getNumBufferedRows() == null - || connectionSettings.getNumBufferedRows() < 10000 - ? 20000 - : (connectionSettings.getNumBufferedRows() * 2)); - BlockingQueue> buffer = new LinkedBlockingDeque<>(bufferSize); - - Runnable populateBufferRunnable = - () -> { // producer thread populating the buffer - Iterable fieldValueLists = - getIterableFieldValueList(results.getRows(), schema); - - boolean hasRows = true; // as we have to process the first page - String pageToken = results.getPageToken(); - JobId jobId = JobId.fromPb(results.getJobReference()); - final TableId destinationTable = queryJobsGetRpc(jobId); - while (hasRows) { - for (FieldValueList fieldValueList : fieldValueLists) { - try { - buffer.put(fieldValueList); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - if (pageToken != null) { // request next page - // get next set of values - TableDataList tabledataList = tableDataListRpc(destinationTable, pageToken); - fieldValueLists = getIterableFieldValueList(tabledataList.getRows(), schema); - hasRows = true; // new page was requested, which is not yet processed - pageToken = tabledataList.getPageToken(); // next page's token - } else { // the current page has been processed and there's no next page - hasRows = false; - } + try { + buffer.put( + new EndOfFieldValueList()); // All the pages has been processed, put this marker + } catch (InterruptedException e) { + throw new BigQueryException(0, e.getMessage(), e); } - buffer.offer( - new EndOfFieldValueList()); // All the pages has been processed, put this marker + queryTaskExecutor.shutdownNow(); // Shutdown the thread pool }; - Thread populateBufferWorker = new Thread(populateBufferRunnable); - populateBufferWorker.start(); // child process to populate the buffer - // This will work for pagination as well, as buffer is getting updated asynchronously - return new BigQueryResultSetImpl>(schema, numRows, buffer); + queryTaskExecutor.execute(populateBufferRunnable); } /* Returns query results using either tabledata.list or the high throughput Read API */ From 2cb80998163b45b301f01312843b094007616b11 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 23 Nov 2021 18:29:58 +0530 Subject: [PATCH 078/277] Cleaned processQueryResponseResults. Removed unused methods. Implemented ThreadPool --- .../main/java/com/google/cloud/bigquery/ConnectionImpl.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 4c85d0742..3d198bb4f 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -336,7 +336,11 @@ private void parseRpcDataAsync( try { pageCache.put(Tuple.of(null, false)); // no further pages } catch (InterruptedException e) { - e.printStackTrace(); + logger.log( + Level.WARNING, + "\n" + Thread.currentThread().getName() + " Interrupted", + e); // Thread might get interrupted while calling the Cancel method, which is + // expected, so logging this instead of throwing the exception back } }; queryTaskExecutor.execute(parseDataTask); From 74f177d98ef3f385c1884c2ab851253558638cbd Mon Sep 17 00:00:00 2001 From: stephwang Date: Tue, 23 Nov 2021 16:13:18 -0500 Subject: [PATCH 079/277] add sessions support and BigQueryResultSetStats interface and impl --- .../cloud/bigquery/BigQueryResultSet.java | 5 ++- .../cloud/bigquery/BigQueryResultSetImpl.java | 13 +++++- .../bigquery/BigQueryResultSetStats.java | 28 +++++++++++++ .../bigquery/BigQueryResultSetStatsImpl.java | 40 +++++++++++++++++++ .../google/cloud/bigquery/ConnectionImpl.java | 21 +++++++++- .../cloud/bigquery/ConnectionSettings.java | 15 +++++++ .../cloud/bigquery/it/ITBigQueryTest.java | 17 ++++++++ 7 files changed, 136 insertions(+), 3 deletions(-) create mode 100644 google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetStats.java create mode 100644 google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetStatsImpl.java diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java index e66c8d7a8..c36e95112 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java @@ -29,6 +29,9 @@ public interface BigQueryResultSet { */ long getTotalRows(); - /*Returns the The underlying ResultSet Implementation*/ + /* Returns the underlying ResultSet Implementation */ ResultSet getResultSet(); + + /* Returns the query statistics associated with this query. */ + BigQueryResultSetStats getBigQueryResultSetStats(); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index 1648ddf35..3d0df7e4b 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -33,12 +33,18 @@ public class BigQueryResultSetImpl implements BigQueryResultSet { private final BlockingQueue buffer; private T cursor; private final ResultSetWrapper underlyingResultSet; + private final BigQueryResultSetStats bigQueryResultSetStats; - public BigQueryResultSetImpl(Schema schema, long totalRows, BlockingQueue buffer) { + public BigQueryResultSetImpl( + Schema schema, + long totalRows, + BlockingQueue buffer, + BigQueryResultSetStats bigQueryResultSetStats) { this.schema = schema; this.totalRows = totalRows; this.buffer = buffer; this.underlyingResultSet = new ResultSetWrapper(); + this.bigQueryResultSetStats = bigQueryResultSetStats; } @Override @@ -402,4 +408,9 @@ public Date getDate(int columnIndex) throws SQLException { return null; // TODO: Implementation for Arrow } } + + @Override + public BigQueryResultSetStats getBigQueryResultSetStats() { + return bigQueryResultSetStats; + } } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetStats.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetStats.java new file mode 100644 index 000000000..cf31642f3 --- /dev/null +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetStats.java @@ -0,0 +1,28 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import com.google.cloud.bigquery.JobStatistics.SessionInfo; + +public interface BigQueryResultSetStats { + + /** Returns detailed statistics for DML statements. */ + DmlStats getDmlStats(); + + /** Returns SessionInfo contains information about the session if this job is part of one. */ + SessionInfo getSessionInfo(); +} diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetStatsImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetStatsImpl.java new file mode 100644 index 000000000..31ca6b14a --- /dev/null +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetStatsImpl.java @@ -0,0 +1,40 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import com.google.cloud.bigquery.JobStatistics.SessionInfo; + +public class BigQueryResultSetStatsImpl implements BigQueryResultSetStats { + + private final DmlStats dmlStats; + private final SessionInfo sessionInfo; + + public BigQueryResultSetStatsImpl(DmlStats dmlStats, SessionInfo sessionInfo) { + this.dmlStats = dmlStats; + this.sessionInfo = sessionInfo; + } + + @Override + public DmlStats getDmlStats() { + return dmlStats; + } + + @Override + public SessionInfo getSessionInfo() { + return sessionInfo; + } +} diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 3d198bb4f..abab420a9 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -216,9 +216,20 @@ private BigQueryResultSet processQueryResponseResults( com.google.api.services.bigquery.model.QueryResponse results) { Schema schema; long numRows; + ; schema = Schema.fromPb(results.getSchema()); numRows = results.getTotalRows().longValue(); + // Get query statistics + DmlStats dmlStats = + results.getDmlStats() == null ? null : DmlStats.fromPb(results.getDmlStats()); + JobStatistics.SessionInfo sessionInfo = + results.getSessionInfo() == null + ? null + : JobStatistics.SessionInfo.fromPb(results.getSessionInfo()); + BigQueryResultSetStats bigQueryResultSetStats = + new BigQueryResultSetStatsImpl(dmlStats, sessionInfo); + // Producer thread for populating the buffer row by row // Keeping the buffersize more than the page size and ensuring it's always a reasonable number int bufferSize = @@ -287,7 +298,8 @@ private BigQueryResultSet processQueryResponseResults( rpcResponseQueue, pageCache, buffer); // spawns a thread to populate the buffer // This will work for pagination as well, as buffer is getting updated asynchronously - return new BigQueryResultSetImpl>(schema, numRows, buffer); + return new BigQueryResultSetImpl>( + schema, numRows, buffer, bigQueryResultSetStats); } /* @@ -562,6 +574,7 @@ private boolean useReadAPI(Long totalRows, Long pageRows) { && totalRows > connectionSettings.getReadClientConnectionConfiguration().getMinResultSize(); } + // Used for job.query endpoint private QueryRequest createQueryRequest( ConnectionSettings connectionSettings, String sql, @@ -588,6 +601,9 @@ private QueryRequest createQueryRequest( if (queryParameters != null) { content.setQueryParameters(queryParameters); } + if (connectionSettings.getCreateSession() != null) { + content.setCreateSession(connectionSettings.getCreateSession()); + } if (labels != null) { content.setLabels(labels); } @@ -685,6 +701,9 @@ private com.google.api.services.bigquery.model.Job createQueryJob( .map(ConnectionProperty.TO_PB_FUNCTION) .collect(Collectors.toList())); } + if (connectionSettings.getCreateSession() != null) { + queryConfigurationPb.setCreateSession(connectionSettings.getCreateSession()); + } if (connectionSettings.getJobTimeoutMs() != null) { configurationPb.setJobTimeoutMs(connectionSettings.getJobTimeoutMs()); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java index e7dca72b8..cdc9dbcc8 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java @@ -125,6 +125,14 @@ public abstract class ConnectionSettings { @Nullable public abstract Boolean getAllowLargeResults(); + /** + * Returns whether to create a new session. + * + * @see Create Sessions + */ + @Nullable + public abstract Boolean getCreateSession(); + /** Returns the range partitioning specification for the table */ @Nullable public abstract RangePartitioning getRangePartitioning(); @@ -325,6 +333,13 @@ public abstract Builder setDestinationEncryptionConfiguration( */ public abstract Builder setAllowLargeResults(Boolean allowLargeResults); + /** + * Sets whether to create a new session. If {@code true} a random session id will be generated + * by BigQuery. If false, runs query with an existing session_id passed in ConnectionProperty, + * otherwise runs query in non-session mode." + */ + public abstract Builder setCreateSession(Boolean createSession); + /** * Range partitioning specification for this table. Only one of timePartitioning and * rangePartitioning should be specified. diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 58cf1373d..938f34ab0 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -51,6 +51,7 @@ import com.google.cloud.bigquery.BigQueryError; import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.BigQueryResultSet; +import com.google.cloud.bigquery.BigQuerySQLException; import com.google.cloud.bigquery.Clustering; import com.google.cloud.bigquery.Connection; import com.google.cloud.bigquery.ConnectionProperty; @@ -2802,6 +2803,22 @@ public void testQuerySessionSupport() throws InterruptedException { assertEquals(sessionId, statisticsWithSession.getSessionInfo().getSessionId()); } + @Test + // TODO: update this testcase when executeUpdate is implemented + public void testExecuteSelectSessionSupport() throws BigQuerySQLException { + String query = "SELECT 17 as foo"; + ConnectionSettings connectionSettings = + ConnectionSettings.newBuilder() + .setDefaultDataset(DatasetId.of(DATASET)) + .setCreateSession(true) + .build(); + Connection connection = bigquery.createConnection(connectionSettings); + BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); + String sessionId = + bigQueryResultSet.getBigQueryResultSetStats().getSessionInfo().getSessionId(); + assertNotNull(sessionId); + } + @Test public void testDmlStatistics() throws InterruptedException { String tableName = TABLE_ID_FASTQUERY.getTable(); From 58ed6d242a4b1ea5d0bf4ca584119831e4e912aa Mon Sep 17 00:00:00 2001 From: stephwang Date: Tue, 23 Nov 2021 18:55:25 -0500 Subject: [PATCH 080/277] add BigQueryResultSetStats to getQueryResults path; refactor jobs.get rpc call method --- .../google/cloud/bigquery/ConnectionImpl.java | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index abab420a9..c9ca7c476 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -216,7 +216,6 @@ private BigQueryResultSet processQueryResponseResults( com.google.api.services.bigquery.model.QueryResponse results) { Schema schema; long numRows; - ; schema = Schema.fromPb(results.getSchema()); numRows = results.getTotalRows().longValue(); @@ -430,8 +429,8 @@ private BigQueryResultSet getQueryResultsWithJobId( */ } - /* Returns the destinationTable from jobId by calling jobs.get API */ - private TableId queryJobsGetRpc(JobId jobId) { + /* Returns Job from jobId by calling the jobs.get API */ + private Job getQueryJobRpc(JobId jobId) { final JobId completeJobId = jobId .setProjectId(bigQueryOptions.getProjectId()) @@ -457,7 +456,12 @@ private TableId queryJobsGetRpc(JobId jobId) { } catch (RetryHelper.RetryHelperException e) { throw BigQueryException.translateAndThrow(e); } - Job job = Job.fromPb(bigQueryOptions.getService(), jobPb); + return Job.fromPb(bigQueryOptions.getService(), jobPb); + } + + /* Returns the destinationTable from jobId by calling jobs.get API */ + private TableId queryJobsGetRpc(JobId jobId) { + Job job = getQueryJobRpc(jobId); return ((QueryJobConfiguration) job.getConfiguration()).getDestinationTable(); } @@ -536,10 +540,19 @@ private BigQueryResultSet processGetQueryResults(JobId jobId, GetQueryResultsRes long numRows = results.getTotalRows() == null ? 0 : results.getTotalRows().longValue(); Schema schema = results.getSchema() == null ? null : Schema.fromPb(results.getSchema()); + // Get query statistics + Job queryJob = getQueryJobRpc(jobId); + JobStatistics.QueryStatistics statistics = queryJob.getStatistics(); + DmlStats dmlStats = statistics.getDmlStats() == null ? null : statistics.getDmlStats(); + JobStatistics.SessionInfo sessionInfo = + statistics.getSessionInfo() == null ? null : statistics.getSessionInfo(); + BigQueryResultSetStats bigQueryResultSetStats = + new BigQueryResultSetStatsImpl(dmlStats, sessionInfo); + // only use this API for the first page of result if (results.getPageToken() == null) { - // return new BigQueryResultSetImpl(schema, numRows, null /* TODO: iterate(cachedFirstPage) - // */); + // TODO: iterate(cachedFirstPage) + // return new BigQueryResultSetImpl(schema, numRows, null, bigQueryResultSetStats); return null; // TODO: Plugin the buffer logic } // use tabledata.list or Read API to fetch subsequent pages of results @@ -574,7 +587,7 @@ private boolean useReadAPI(Long totalRows, Long pageRows) { && totalRows > connectionSettings.getReadClientConnectionConfiguration().getMinResultSize(); } - // Used for job.query endpoint + // Used for job.query API endpoint private QueryRequest createQueryRequest( ConnectionSettings connectionSettings, String sql, @@ -614,6 +627,7 @@ private QueryRequest createQueryRequest( return content; } + // Used for jobs.getQueryResults API endpoint private com.google.api.services.bigquery.model.Job createQueryJob( String sql, ConnectionSettings connectionSettings, From 5fad3e53df73db9b1b7b3996ee51f0ab6116d6be Mon Sep 17 00:00:00 2001 From: stephwang Date: Wed, 1 Dec 2021 12:40:23 -0500 Subject: [PATCH 081/277] feat: add support for BI Engine Statistics fixes b/205146044 --- .../google/cloud/bigquery/BiEngineReason.java | 92 ++++++++++++++++ .../google/cloud/bigquery/BiEngineStats.java | 101 ++++++++++++++++++ .../google/cloud/bigquery/JobStatistics.java | 22 ++++ .../cloud/bigquery/JobStatisticsTest.java | 12 +++ .../cloud/bigquery/it/ITBigQueryTest.java | 7 ++ 5 files changed, 234 insertions(+) create mode 100644 google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BiEngineReason.java create mode 100644 google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BiEngineStats.java diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BiEngineReason.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BiEngineReason.java new file mode 100644 index 000000000..e682ba246 --- /dev/null +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BiEngineReason.java @@ -0,0 +1,92 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import com.google.auto.value.AutoValue; +import java.io.Serializable; +import javax.annotation.Nullable; + +@AutoValue +public abstract class BiEngineReason implements Serializable { + + @AutoValue.Builder + public abstract static class Builder { + + /** + * High-level BI Engine reason for partial or disabled acceleration. + * + * @param code code or {@code null} for none + */ + public abstract Builder setCode(String code); + + /** + * Free form human-readable reason for partial or disabled acceleration. + * + * @param message message or {@code null} for none + */ + public abstract Builder setMessage(String message); + + /** Creates a {@code BiEngineReason} object. */ + public abstract BiEngineReason build(); + } + + /** + * High-level BI Engine reason for partial or disabled acceleration. + * + * @return value or {@code null} for none + */ + @Nullable + public abstract String getCode(); + + /** + * Free form human-readable reason for partial or disabled acceleration. + * + * @return value or {@code null} for none + */ + @Nullable + public abstract String getMessage(); + + public abstract Builder toBuilder(); + + public static Builder newBuilder() { + return new AutoValue_BiEngineReason.Builder(); + } + + com.google.api.services.bigquery.model.BiEngineReason toPb() { + com.google.api.services.bigquery.model.BiEngineReason biEngineReasonPb = + new com.google.api.services.bigquery.model.BiEngineReason(); + if (getCode() != null) { + biEngineReasonPb.setCode(getCode()); + } + if (getMessage() != null) { + biEngineReasonPb.setMessage(getMessage()); + } + return biEngineReasonPb; + } + + static BiEngineReason fromPb( + com.google.api.services.bigquery.model.BiEngineReason biEngineReasonPb) { + Builder builder = newBuilder(); + if (biEngineReasonPb.getCode() != null) { + builder.setCode(biEngineReasonPb.getCode()); + } + if (biEngineReasonPb.getMessage() != null) { + builder.setMessage(biEngineReasonPb.getMessage()); + } + return builder.build(); + } +} diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BiEngineStats.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BiEngineStats.java new file mode 100644 index 000000000..34d6c4326 --- /dev/null +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BiEngineStats.java @@ -0,0 +1,101 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import com.google.api.services.bigquery.model.BiEngineStatistics; +import com.google.auto.value.AutoValue; +import java.io.Serializable; +import java.util.List; +import java.util.stream.Collectors; +import javax.annotation.Nullable; + +/** BIEngineStatistics contains query statistics specific to the use of BI Engine. */ +@AutoValue +public abstract class BiEngineStats implements Serializable { + + @AutoValue.Builder + public abstract static class Builder { + /** + * Specifies which mode of BI Engine acceleration was performed (if any). + * + * @param biEngineMode biEngineMode or {@code null} for none + */ + public abstract Builder setBiEngineMode(String biEngineMode); + + /** + * In case of DISABLED or PARTIAL bi_engine_mode, these contain the explanatory reasons as to + * why BI Engine could not accelerate. In case the full query was accelerated, this field is not + * populated. + * + * @param biEngineReasons biEngineReasons or {@code null} for none + */ + public abstract Builder setBiEngineReasons(List biEngineReasons); + + /** Creates a @code BiEngineStats} object. */ + public abstract BiEngineStats build(); + } + + /** + * Specifies which mode of BI Engine acceleration was performed (if any). + * + * @return value or {@code null} for none + */ + @Nullable + public abstract String getBiEngineMode(); + + /** + * In case of DISABLED or PARTIAL bi_engine_mode, these contain the explanatory reasons as to why + * BI Engine could not accelerate. In case the full query was accelerated, this field is not + * populated. + * + * @return value or {@code null} for none + */ + @Nullable + public abstract List getBiEngineReasons(); + + public abstract Builder toBuilder(); + + public static Builder newBuilder() { + return new AutoValue_BiEngineStats.Builder(); + } + + BiEngineStatistics toPb() { + BiEngineStatistics biEngineStatisticsPb = new BiEngineStatistics(); + if (getBiEngineMode() != null) { + biEngineStatisticsPb.setBiEngineMode(getBiEngineMode()); + } + if (getBiEngineReasons() != null) { + biEngineStatisticsPb.setBiEngineReasons( + getBiEngineReasons().stream().map(BiEngineReason::toPb).collect(Collectors.toList())); + } + return biEngineStatisticsPb; + } + + static BiEngineStats fromPb(BiEngineStatistics biEngineStatisticsPb) { + Builder builder = newBuilder(); + if (biEngineStatisticsPb.getBiEngineMode() != null) { + builder.setBiEngineMode(biEngineStatisticsPb.getBiEngineMode()); + } + if (biEngineStatisticsPb.getBiEngineReasons() != null) { + builder.setBiEngineReasons( + biEngineStatisticsPb.getBiEngineReasons().stream() + .map(BiEngineReason::fromPb) + .collect(Collectors.toList())); + } + return builder.build(); + } +} diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java index 999f8d2fe..ab9fdabb3 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java @@ -321,6 +321,7 @@ public static class QueryStatistics extends JobStatistics { private static final long serialVersionUID = 7539354109226732353L; + private final BiEngineStats biEngineStats; private final Integer billingTier; private final Boolean cacheHit; private final String ddlOperationPerformed; @@ -402,6 +403,7 @@ public static StatementType[] values() { static final class Builder extends JobStatistics.Builder { + private BiEngineStats biEngineStats; private Integer billingTier; private Boolean cacheHit; private String ddlOperationPerformed; @@ -425,6 +427,10 @@ private Builder() {} private Builder(com.google.api.services.bigquery.model.JobStatistics statisticsPb) { super(statisticsPb); if (statisticsPb.getQuery() != null) { + if (statisticsPb.getQuery().getBiEngineStatistics() != null) { + this.biEngineStats = + BiEngineStats.fromPb(statisticsPb.getQuery().getBiEngineStatistics()); + } this.billingTier = statisticsPb.getQuery().getBillingTier(); this.cacheHit = statisticsPb.getQuery().getCacheHit(); this.ddlOperationPerformed = statisticsPb.getQuery().getDdlOperationPerformed(); @@ -468,6 +474,11 @@ private Builder(com.google.api.services.bigquery.model.JobStatistics statisticsP } } + Builder setBiEngineStats(BiEngineStats biEngineStats) { + this.biEngineStats = biEngineStats; + return self(); + } + Builder setBillingTier(Integer billingTier) { this.billingTier = billingTier; return self(); @@ -566,6 +577,7 @@ QueryStatistics build() { private QueryStatistics(Builder builder) { super(builder); + this.biEngineStats = builder.biEngineStats; this.billingTier = builder.billingTier; this.cacheHit = builder.cacheHit; this.ddlOperationPerformed = builder.ddlOperationPerformed; @@ -585,6 +597,11 @@ private QueryStatistics(Builder builder) { this.schema = builder.schema; } + /** Returns query statistics specific to the use of BI Engine. */ + public BiEngineStats getBiEngineStats() { + return biEngineStats; + } + /** Returns the billing tier for the job. */ public Integer getBillingTier() { return billingTier; @@ -701,6 +718,7 @@ public Schema getSchema() { @Override ToStringHelper toStringHelper() { return super.toStringHelper() + .add("biEngineStats", biEngineStats) .add("billingTier", billingTier) .add("cacheHit", cacheHit) .add("totalBytesBilled", totalBytesBilled) @@ -722,6 +740,7 @@ public final boolean equals(Object obj) { public final int hashCode() { return Objects.hash( baseHashCode(), + biEngineStats, billingTier, cacheHit, totalBytesBilled, @@ -733,6 +752,9 @@ public final int hashCode() { @Override com.google.api.services.bigquery.model.JobStatistics toPb() { JobStatistics2 queryStatisticsPb = new JobStatistics2(); + if (biEngineStats != null) { + queryStatisticsPb.setBiEngineStatistics(biEngineStats.toPb()); + } queryStatisticsPb.setBillingTier(billingTier); queryStatisticsPb.setCacheHit(cacheHit); queryStatisticsPb.setDdlOperationPerformed(ddlOperationPerformed); diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/JobStatisticsTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/JobStatisticsTest.java index 87fae41b5..0dad46e9e 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/JobStatisticsTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/JobStatisticsTest.java @@ -36,6 +36,16 @@ public class JobStatisticsTest { + private static final BiEngineReason BI_ENGINE_REASON = + BiEngineReason.newBuilder() + .setMessage("Detected unsupported join type") + .setCode("UNSUPPORTED_SQL_TEXT") + .build(); + private static final BiEngineStats BI_ENGINE_STATS = + BiEngineStats.newBuilder() + .setBiEngineReasons(ImmutableList.of(BI_ENGINE_REASON)) + .setBiEngineMode("DISABLED") + .build(); private static final Integer BILLING_TIER = 42; private static final Boolean CACHE_HIT = true; private static final String DDL_OPERATION_PERFORMED = "SKIP"; @@ -154,6 +164,7 @@ public class JobStatisticsTest { .setCreationTimestamp(CREATION_TIME) .setEndTime(END_TIME) .setStartTime(START_TIME) + .setBiEngineStats(BI_ENGINE_STATS) .setBillingTier(BILLING_TIER) .setCacheHit(CACHE_HIT) .setDDLOperationPerformed(DDL_OPERATION_PERFORMED) @@ -246,6 +257,7 @@ public void testBuilder() { assertEquals(CREATION_TIME, QUERY_STATISTICS.getCreationTime()); assertEquals(START_TIME, QUERY_STATISTICS.getStartTime()); assertEquals(END_TIME, QUERY_STATISTICS.getEndTime()); + assertEquals(BI_ENGINE_STATS, QUERY_STATISTICS.getBiEngineStats()); assertEquals(BILLING_TIER, QUERY_STATISTICS.getBillingTier()); assertEquals(CACHE_HIT, QUERY_STATISTICS.getCacheHit()); assertEquals(DDL_OPERATION_PERFORMED, QUERY_STATISTICS.getDdlOperationPerformed()); diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 938f34ab0..5165e0d52 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -3451,6 +3451,13 @@ public void testQueryJob() throws InterruptedException, TimeoutException { assertTrue(bigquery.delete(destinationTable)); Job queryJob = bigquery.getJob(remoteJob.getJobId()); JobStatistics.QueryStatistics statistics = queryJob.getStatistics(); + assertNotNull(statistics.getBiEngineStats()); + assertEquals(statistics.getBiEngineStats().getBiEngineMode(), "DISABLED"); + assertEquals( + statistics.getBiEngineStats().getBiEngineReasons().get(0).getCode(), "OTHER_REASON"); + assertEquals( + statistics.getBiEngineStats().getBiEngineReasons().get(0).getMessage(), + "Query output to destination table is not supported."); assertNotNull(statistics.getQueryPlan()); } From 73e05fd8e6f39d196cda68c64b9f068923455e54 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 7 Dec 2021 11:48:35 +0530 Subject: [PATCH 082/277] Updated pageCacheSize and buffer's size between a MIN and MAX bound. --- .../java/com/google/cloud/bigquery/ConnectionImpl.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index c9ca7c476..8f7a30c4d 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -154,7 +154,7 @@ private BigQueryResultSet queryRpc(final String projectId, final QueryRequest qu // Query finished running and we can paginate all the results if (results.getJobComplete() && results.getSchema() != null) { - return processQueryResponseResults(results); // processQueryResponseResults(results); + return processQueryResponseResults(results); } else { // Query is long running (> 10s) and hasn't completed yet, or query completed but didn't // return the schema, fallback to jobs.insert path. Some operations don't return the schema @@ -184,10 +184,10 @@ public FieldValueList apply(TableRow rowPb) { // Determines the optimal number of caches pages to improve read performance private int getPageCacheSize(Long numBufferedRows, long numRows, Schema schema) { - final int MIN_CACHE_SIZE = 2; // Min number of pages in the page size + final int MIN_CACHE_SIZE = 3; // Min number of pages in the page size final int MAX_CACHE_SIZE = 20; // //Min number of pages in the page size int columnsRead = schema.getFields().size(); - int numCachedPages = 10; // default page size + int numCachedPages; long numCachedRows = numBufferedRows == null ? 0 : numBufferedRows.longValue(); // TODO: Further enhance this logic @@ -236,7 +236,7 @@ private BigQueryResultSet processQueryResponseResults( (connectionSettings.getNumBufferedRows() == null || connectionSettings.getNumBufferedRows() < 10000 ? 20000 - : (connectionSettings.getNumBufferedRows() * 2)); + : Math.min(connectionSettings.getNumBufferedRows() * 2, 100000));//ensuring that the buffer has values between 20K and 100K BlockingQueue> buffer = new LinkedBlockingDeque<>( bufferSize); // this keeps the deserialized records at the row level, which will be From 86bdf13b12b70b382f756b6a903ad5060bf9aa38 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Wed, 8 Dec 2021 00:12:21 +0000 Subject: [PATCH 083/277] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b7643a88d..21b58f909 100644 --- a/README.md +++ b/README.md @@ -58,13 +58,13 @@ implementation 'com.google.cloud:google-cloud-bigquery' If you are using Gradle without BOM, add this to your dependencies ```Groovy -implementation 'com.google.cloud:google-cloud-bigquery:2.4.1' +implementation 'com.google.cloud:google-cloud-bigquery:2.5.1' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.4.1" +libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.5.1" ``` ## Authentication From bcf6dfde8b551a44c8e4471b962b8cefd4fa9d8c Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 9 Dec 2021 08:33:41 +0530 Subject: [PATCH 084/277] Refactored processQueryResponseResults to avoid code duplicity. Added upper bound for the buffer size --- .../google/cloud/bigquery/ConnectionImpl.java | 76 ++++++++++++------- 1 file changed, 49 insertions(+), 27 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 8f7a30c4d..81de02be7 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -236,7 +236,9 @@ private BigQueryResultSet processQueryResponseResults( (connectionSettings.getNumBufferedRows() == null || connectionSettings.getNumBufferedRows() < 10000 ? 20000 - : Math.min(connectionSettings.getNumBufferedRows() * 2, 100000));//ensuring that the buffer has values between 20K and 100K + : Math.min( + connectionSettings.getNumBufferedRows() * 2, + 100000)); // ensuring that the buffer has values between 20K and 100K BlockingQueue> buffer = new LinkedBlockingDeque<>( bufferSize); // this keeps the deserialized records at the row level, which will be @@ -255,8 +257,27 @@ private BigQueryResultSet processQueryResponseResults( schema)); // this keeps the raw RPC response JobId jobId = JobId.fromPb(results.getJobReference()); - final TableId destinationTable = queryJobsGetRpc(jobId); + runNextPageTaskAsync(results, queryJobsGetRpc(jobId), rpcResponseQueue); + parseRpcDataAsync( + results, + schema, + pageCache, + rpcResponseQueue); // parses data on a separate thread, thus maximising processing + // throughput + + populateBufferAsync( + rpcResponseQueue, pageCache, buffer); // spawns a thread to populate the buffer + + // This will work for pagination as well, as buffer is getting updated asynchronously + return new BigQueryResultSetImpl>( + schema, numRows, buffer, bigQueryResultSetStats); + } + + private void runNextPageTaskAsync( + com.google.api.services.bigquery.model.QueryResponse results, + TableId destinationTable, + BlockingQueue> rpcResponseQueue) { // This thread makes the RPC calls and paginates Runnable nextPageTask = () -> { @@ -285,20 +306,6 @@ private BigQueryResultSet processQueryResponseResults( } }; queryTaskExecutor.execute(nextPageTask); - - parseRpcDataAsync( - results, - schema, - pageCache, - rpcResponseQueue); // parses data on a separate thread, thus maximising processing - // throughput - - populateBufferAsync( - rpcResponseQueue, pageCache, buffer); // spawns a thread to populate the buffer - - // This will work for pagination as well, as buffer is getting updated asynchronously - return new BigQueryResultSetImpl>( - schema, numRows, buffer, bigQueryResultSetStats); } /* @@ -421,12 +428,9 @@ private void populateBufferAsync( private BigQueryResultSet getQueryResultsWithJobId( long totalRows, long pageRows, Schema schema, JobId jobId) { TableId destinationTable = queryJobsGetRpc(jobId); - - return null; // use processQueryResponseResults(); for remaining pages - /* return useReadAPI(totalRows, pageRows) - ? highThroughPutRead(destinationTable) - : null; // plugin tableDataListRpc(destinationTable, schema, null);, Use processQueryResponseResults ? - */ + return useReadAPI(totalRows, pageRows) + ? highThroughPutRead(destinationTable) + : getQueryResultsRpc(jobId); } /* Returns Job from jobId by calling the jobs.get API */ @@ -498,8 +502,7 @@ private BigQueryResultSet highThroughPutRead(TableId destinationTable) { } /* Returns results of the query associated with the provided job using jobs.getQueryResults API */ - private BigQueryResultSet getQueryResultsRpc( - JobId jobId) { // TODO(prasmish) temp: This is a slower endpoint + private BigQueryResultSet getQueryResultsRpc(JobId jobId) { JobId completeJobId = jobId .setProjectId(bigQueryOptions.getProjectId()) @@ -508,6 +511,11 @@ private BigQueryResultSet getQueryResultsRpc( ? bigQueryOptions.getLocation() : jobId.getLocation()); try { + + // Question: Can we use tableData.list endpoint instead? + // Table data.list doesn't have destination table! + // Cant be + GetQueryResultsResponse results = BigQueryRetryHelper.runWithRetries( () -> @@ -549,16 +557,30 @@ private BigQueryResultSet processGetQueryResults(JobId jobId, GetQueryResultsRes BigQueryResultSetStats bigQueryResultSetStats = new BigQueryResultSetStatsImpl(dmlStats, sessionInfo); - // only use this API for the first page of result + /* // only use this API for the first page of result if (results.getPageToken() == null) { // TODO: iterate(cachedFirstPage) + return processQueryResponseResults(results); // return new BigQueryResultSetImpl(schema, numRows, null, bigQueryResultSetStats); return null; // TODO: Plugin the buffer logic - } + }*/ + + // results + /* for (TableRow tr : results.getRows()) { + // tr.getF(). + FieldValue.fromPb(tr.getF(), null); + }*/ + // results.getp + + // We need com.google.api.services.bigquery.model.QueryResponse to invoke + // processQueryResponseResults + // processQueryResponseResults(res); + // TODO(prasmish) : add the processQueryResponseResults logic // use tabledata.list or Read API to fetch subsequent pages of results long totalRows = results.getTotalRows() == null ? 0 : results.getTotalRows().longValue(); long pageRows = results.getRows().size(); - return getQueryResultsWithJobId(totalRows, pageRows, schema, jobId); + + return null; // getQueryResultsWithJobId(totalRows, pageRows, schema, jobId); } private boolean isFastQuerySupported() { From 1d900b51117e55f7585810601ef83e08a3a6155a Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 9 Dec 2021 09:49:49 +0530 Subject: [PATCH 085/277] Implemented processGetQueryResponseResults(results, jobId) & Refactored the code --- .../google/cloud/bigquery/ConnectionImpl.java | 119 ++++++++++++++---- 1 file changed, 95 insertions(+), 24 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 81de02be7..4e564c70a 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -212,6 +212,69 @@ private int getPageCacheSize(Long numBufferedRows, long numRows, Schema schema) MAX_CACHE_SIZE)); // numCachedPages should be between the defined min and max } + /* This method processed the first page of GetQueryResultsResponse and then it uses tabledata.list */ + private BigQueryResultSet processGetQueryResponseResults( + GetQueryResultsResponse results, JobId completeJobId) { + Schema schema; + long numRows; + schema = Schema.fromPb(results.getSchema()); + numRows = results.getTotalRows().longValue(); + + // Get query statistics + Job queryJob = getQueryJobRpc(completeJobId); + JobStatistics.QueryStatistics statistics = queryJob.getStatistics(); + DmlStats dmlStats = statistics.getDmlStats() == null ? null : statistics.getDmlStats(); + JobStatistics.SessionInfo sessionInfo = + statistics.getSessionInfo() == null ? null : statistics.getSessionInfo(); + BigQueryResultSetStats bigQueryResultSetStats = + new BigQueryResultSetStatsImpl(dmlStats, sessionInfo); + + // Producer thread for populating the buffer row by row + // Keeping the buffersize more than the page size and ensuring it's always a reasonable number + int bufferSize = + (int) + (connectionSettings.getNumBufferedRows() == null + || connectionSettings.getNumBufferedRows() < 10000 + ? 20000 + : Math.min( + connectionSettings.getNumBufferedRows() * 2, + 100000)); // ensuring that the buffer has values between 20K and 100K + BlockingQueue> buffer = + new LinkedBlockingDeque<>( + bufferSize); // this keeps the deserialized records at the row level, which will be + // consumed by the BQResultSet + BlockingQueue, Boolean>> pageCache = + new LinkedBlockingDeque<>( + getPageCacheSize( + connectionSettings.getNumBufferedRows(), + numRows, + schema)); // this keeps the parsed FieldValueLists + BlockingQueue> rpcResponseQueue = + new LinkedBlockingDeque<>( + getPageCacheSize( + connectionSettings.getNumBufferedRows(), + numRows, + schema)); // this keeps the raw RPC response + + JobId jobId = JobId.fromPb(results.getJobReference()); + + runNextPageTaskAsync(results.getPageToken(), queryJobsGetRpc(jobId), rpcResponseQueue); + + parseRpcDataAsync( + results.getRows(), + schema, + pageCache, + rpcResponseQueue); // parses data on a separate thread, thus maximising processing + // throughput + + populateBufferAsync( + rpcResponseQueue, pageCache, buffer); // spawns a thread to populate the buffer + + // This will work for pagination as well, as buffer is getting updated asynchronously + return new BigQueryResultSetImpl>( + schema, numRows, buffer, bigQueryResultSetStats); + } + private BigQueryResultSet processQueryResponseResults( com.google.api.services.bigquery.model.QueryResponse results) { Schema schema; @@ -257,10 +320,10 @@ private BigQueryResultSet processQueryResponseResults( schema)); // this keeps the raw RPC response JobId jobId = JobId.fromPb(results.getJobReference()); - runNextPageTaskAsync(results, queryJobsGetRpc(jobId), rpcResponseQueue); + runNextPageTaskAsync(results.getPageToken(), queryJobsGetRpc(jobId), rpcResponseQueue); parseRpcDataAsync( - results, + results.getRows(), schema, pageCache, rpcResponseQueue); // parses data on a separate thread, thus maximising processing @@ -275,13 +338,13 @@ private BigQueryResultSet processQueryResponseResults( } private void runNextPageTaskAsync( - com.google.api.services.bigquery.model.QueryResponse results, + String firstPageToken, TableId destinationTable, BlockingQueue> rpcResponseQueue) { // This thread makes the RPC calls and paginates Runnable nextPageTask = () -> { - String pageToken = results.getPageToken(); + String pageToken = firstPageToken; // results.getPageToken(); try { while (pageToken != null) { // paginate for non null token if (Thread.currentThread().isInterrupted() @@ -312,15 +375,15 @@ private void runNextPageTaskAsync( This method takes TableDataList from rpcResponseQueue and populates pageCache with FieldValueList */ private void parseRpcDataAsync( - com.google.api.services.bigquery.model.QueryResponse results, + // com.google.api.services.bigquery.model.QueryResponse results, + List tableRows, Schema schema, BlockingQueue, Boolean>> pageCache, BlockingQueue> rpcResponseQueue) { // parse and put the first page in the pageCache before the other pages are parsed from the RPC // calls - Iterable firstFieldValueLists = - getIterableFieldValueList(results.getRows(), schema); + Iterable firstFieldValueLists = getIterableFieldValueList(tableRows, schema); try { pageCache.put( Tuple.of(firstFieldValueLists, true)); // this is the first page which we have received. @@ -538,12 +601,16 @@ private BigQueryResultSet getQueryResultsRpc(JobId jobId) { // with the case where there there is a HTTP error throw new BigQueryException(bigQueryErrors); } - return processGetQueryResults(jobId, results); + // return processGetQueryResults(jobId, results); + return processGetQueryResponseResults(results, jobId); } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); } } + /* + //TODO(prasmish): We have wired processQueryResponseResults(results, jobId) method instead, delete this after code review + private BigQueryResultSet processGetQueryResults(JobId jobId, GetQueryResultsResponse results) { long numRows = results.getTotalRows() == null ? 0 : results.getTotalRows().longValue(); Schema schema = results.getSchema() == null ? null : Schema.fromPb(results.getSchema()); @@ -557,31 +624,35 @@ private BigQueryResultSet processGetQueryResults(JobId jobId, GetQueryResultsRes BigQueryResultSetStats bigQueryResultSetStats = new BigQueryResultSetStatsImpl(dmlStats, sessionInfo); - /* // only use this API for the first page of result - if (results.getPageToken() == null) { - // TODO: iterate(cachedFirstPage) - return processQueryResponseResults(results); - // return new BigQueryResultSetImpl(schema, numRows, null, bigQueryResultSetStats); - return null; // TODO: Plugin the buffer logic - }*/ - - // results - /* for (TableRow tr : results.getRows()) { - // tr.getF(). - FieldValue.fromPb(tr.getF(), null); - }*/ + */ + /* // only use this API for the first page of result + if (results.getPageToken() == null) { + + return processQueryResponseResults(results); + // return new BigQueryResultSetImpl(schema, numRows, null, bigQueryResultSetStats); + return null; // + }*/ + /* + + // results + */ + /* for (TableRow tr : results.getRows()) { + // tr.getF(). + FieldValue.fromPb(tr.getF(), null); + }*/ + /* // results.getp // We need com.google.api.services.bigquery.model.QueryResponse to invoke // processQueryResponseResults - // processQueryResponseResults(res); - // TODO(prasmish) : add the processQueryResponseResults logic + // processQueryResponseResults(results); + // use tabledata.list or Read API to fetch subsequent pages of results long totalRows = results.getTotalRows() == null ? 0 : results.getTotalRows().longValue(); long pageRows = results.getRows().size(); return null; // getQueryResultsWithJobId(totalRows, pageRows, schema, jobId); - } + }*/ private boolean isFastQuerySupported() { // TODO: add regex logic to check for scripting From 40a9226e520b17bb8da447064e5444b6532b0fcb Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 21 Dec 2021 14:00:00 +0530 Subject: [PATCH 086/277] Implemented BigQueryDryRunResultImpl --- .../bigquery/BigQueryDryRunResultImpl.java | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResultImpl.java diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResultImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResultImpl.java new file mode 100644 index 000000000..76d9174ec --- /dev/null +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResultImpl.java @@ -0,0 +1,41 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import com.google.api.services.bigquery.model.QueryParameter; +import java.util.List; + +public class BigQueryDryRunResultImpl implements BigQueryDryRunResult { + private Schema schema; + private List queryParameters; + + BigQueryDryRunResultImpl( + Schema schema, List queryParameters) { // Package-Private access + this.schema = schema; + this.queryParameters = queryParameters; + } + + @Override + public Schema getSchema() throws BigQuerySQLException { + return schema; + } + + @Override + public List getQueryParameters() throws BigQuerySQLException { + return queryParameters; + } +} From 26d55e3fa74161e1e434c50b3b100c7ddd0892de Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 21 Dec 2021 14:00:59 +0530 Subject: [PATCH 087/277] implemented dryRun --- .../java/com/google/cloud/bigquery/ConnectionImpl.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 4e564c70a..4cc1281df 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -77,9 +77,13 @@ public synchronized Boolean cancel() throws BigQuerySQLException { @Override public BigQueryDryRunResult dryRun(String sql) throws BigQuerySQLException { - // TODO: run a dummy query using tabaledata.list end point ? What are the query params - // jobs.query or jobs.getqueuryresults queryrequestinfo.java - return null; + QueryJobConfiguration queryConfig = + QueryJobConfiguration.newBuilder(sql).setDryRun(true).setUseQueryCache(false).build(); + Job job = BigQueryOptions.getDefaultInstance().getService().create(JobInfo.of(queryConfig)); + JobStatistics.QueryStatistics statistics = job.getStatistics(); + + return new BigQueryDryRunResultImpl( + statistics.getSchema(), queryConfig.toPb().getQuery().getQueryParameters()); } @Override From 8c143921acd78fe50ea38752be1f75e1f80586bf Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 28 Dec 2021 10:44:25 +0530 Subject: [PATCH 088/277] Implemented execute select logic when the FastQuery is not Supported --- .../google/cloud/bigquery/ConnectionImpl.java | 90 +++++++++++++++---- 1 file changed, 72 insertions(+), 18 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 4cc1281df..87ab7a733 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -78,12 +78,47 @@ public synchronized Boolean cancel() throws BigQuerySQLException { @Override public BigQueryDryRunResult dryRun(String sql) throws BigQuerySQLException { QueryJobConfiguration queryConfig = - QueryJobConfiguration.newBuilder(sql).setDryRun(true).setUseQueryCache(false).build(); - Job job = BigQueryOptions.getDefaultInstance().getService().create(JobInfo.of(queryConfig)); - JobStatistics.QueryStatistics statistics = job.getStatistics(); + QueryJobConfiguration.newBuilder(sql).setDryRun(true).build(); + // Job job = BigQueryOptions.getDefaultInstance().getService().create(JobInfo.of(queryConfig)); + /*//TODO: WIP implementation + JobId jobId = JobId.of(UUID.randomUUID().toString()); + Job job = BigQueryOptions.getDefaultInstance().getService().create(JobInfo.newBuilder(queryConfig).setJobId(jobId).build()); + + try { + if(!job.isDone()){//wait if job is still running. TODO(prasmish): Doublecheck if this check is required + job = job.waitFor(); + } + + // response.get + } catch (InterruptedException e) { + e.printStackTrace(); + } + JobStatistics.QueryStatistics statistics = job.getStatistics(); + try { + TableResult tr = job.getQueryResults(); + // job.getBigQuery().getQueryResults(). + // tr.get + // job.getConfiguration().g + } catch (InterruptedException e) { + e.printStackTrace(); + } + com.google.cloud.bigquery.QueryResponse response = BigQueryOptions.getDefaultInstance().getService().getQueryResults(jobId); + JobId jobId = JobId.fromPb(response.getJobReference()); + + com.google.api.services.bigquery.model.QueryResponse results; + results.getJobReference().getJobId() + + JobId jobIdd = JobId.fromPb(results.getJobReference()).get; + //response. + + JobConfigurationQuery d = queryConfig.toPb().getQuery(); + List qp = d.getQueryParameters(); + //response. + return new BigQueryDryRunResultImpl( + response.getSchema(), queryConfig.toPb().getQuery().getQueryParameters()); - return new BigQueryDryRunResultImpl( - statistics.getSchema(), queryConfig.toPb().getQuery().getQueryParameters()); + */ + return null; } @Override @@ -97,7 +132,10 @@ public BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException { // use jobs.insert otherwise com.google.api.services.bigquery.model.Job queryJob = createQueryJob(sql, connectionSettings, null, null); - return null; // TODO getQueryResultsRpc(JobId.fromPb(queryJob.getJobReference())); + JobId jobId = JobId.fromPb(queryJob.getJobReference()); + GetQueryResultsResponse firstPage = getQueryResultsFirstPage(jobId); + return getQueryResultsWithJobId( + firstPage.getTotalRows().longValue(), firstPage.getRows().size(), null, jobId, firstPage); } @Override @@ -114,7 +152,10 @@ public BigQueryResultSet executeSelect( // use jobs.insert otherwise com.google.api.services.bigquery.model.Job queryJob = createQueryJob(sql, connectionSettings, parameters, labels); - return null; // TODO getQueryResultsRpc(JobId.fromPb(queryJob.getJobReference())); + JobId jobId = JobId.fromPb(queryJob.getJobReference()); + GetQueryResultsResponse firstPage = getQueryResultsFirstPage(jobId); + return getQueryResultsWithJobId( + firstPage.getTotalRows().longValue(), firstPage.getRows().size(), null, jobId, firstPage); } static class EndOfFieldValueList @@ -167,7 +208,7 @@ private BigQueryResultSet queryRpc(final String projectId, final QueryRequest qu long totalRows = results.getTotalRows().longValue(); long pageRows = results.getRows().size(); JobId jobId = JobId.fromPb(results.getJobReference()); - return getQueryResultsWithJobId(totalRows, pageRows, null, jobId); + return getQueryResultsWithJobId(totalRows, pageRows, null, jobId, null); } } @@ -216,6 +257,7 @@ private int getPageCacheSize(Long numBufferedRows, long numRows, Schema schema) MAX_CACHE_SIZE)); // numCachedPages should be between the defined min and max } + // TODO: Write a IT with order by query /* This method processed the first page of GetQueryResultsResponse and then it uses tabledata.list */ private BigQueryResultSet processGetQueryResponseResults( GetQueryResultsResponse results, JobId completeJobId) { @@ -493,11 +535,15 @@ private void populateBufferAsync( /* Returns query results using either tabledata.list or the high throughput Read API */ private BigQueryResultSet getQueryResultsWithJobId( - long totalRows, long pageRows, Schema schema, JobId jobId) { + long totalRows, + long pageRows, + Schema schema, + JobId jobId, + GetQueryResultsResponse firstPage) { TableId destinationTable = queryJobsGetRpc(jobId); return useReadAPI(totalRows, pageRows) ? highThroughPutRead(destinationTable) - : getQueryResultsRpc(jobId); + : getQueryResultsRpc(jobId, firstPage); } /* Returns Job from jobId by calling the jobs.get API */ @@ -569,7 +615,21 @@ private BigQueryResultSet highThroughPutRead(TableId destinationTable) { } /* Returns results of the query associated with the provided job using jobs.getQueryResults API */ - private BigQueryResultSet getQueryResultsRpc(JobId jobId) { + private BigQueryResultSet getQueryResultsRpc(JobId jobId, GetQueryResultsResponse firstPage) { + try { + // paginate using multi threaded logic and return BigQueryResultSet + if (firstPage == null) { + return processGetQueryResponseResults(getQueryResultsFirstPage(jobId), jobId); + } else { // we already received the firstpage using the jobId + return processGetQueryResponseResults(firstPage, jobId); + } + } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) { + throw BigQueryException.translateAndThrow(e); + } + } + + /*Returns just the first page of GetQueryResultsResponse using the jobId*/ + private GetQueryResultsResponse getQueryResultsFirstPage(JobId jobId) { JobId completeJobId = jobId .setProjectId(bigQueryOptions.getProjectId()) @@ -578,11 +638,6 @@ private BigQueryResultSet getQueryResultsRpc(JobId jobId) { ? bigQueryOptions.getLocation() : jobId.getLocation()); try { - - // Question: Can we use tableData.list endpoint instead? - // Table data.list doesn't have destination table! - // Cant be - GetQueryResultsResponse results = BigQueryRetryHelper.runWithRetries( () -> @@ -605,8 +660,7 @@ private BigQueryResultSet getQueryResultsRpc(JobId jobId) { // with the case where there there is a HTTP error throw new BigQueryException(bigQueryErrors); } - // return processGetQueryResults(jobId, results); - return processGetQueryResponseResults(results, jobId); + return results; } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); } From ca9b7d41e7ccace259ca50e8990644c18b6a9bec Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 29 Dec 2021 14:53:13 +0530 Subject: [PATCH 089/277] added null check at useReadAPI --- .../google/cloud/bigquery/ConnectionImpl.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 87ab7a733..d70430af2 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -731,11 +731,17 @@ private boolean isFastQuerySupported() { private boolean useReadAPI(Long totalRows, Long pageRows) { long resultRatio = totalRows / pageRows; - return resultRatio - > connectionSettings - .getReadClientConnectionConfiguration() - .getTotalToPageRowCountRatio() - && totalRows > connectionSettings.getReadClientConnectionConfiguration().getMinResultSize(); + if (connectionSettings.getReadClientConnectionConfiguration() + != null) { // Adding a null check to avoid NPE + return resultRatio + > connectionSettings + .getReadClientConnectionConfiguration() + .getTotalToPageRowCountRatio() + && totalRows + > connectionSettings.getReadClientConnectionConfiguration().getMinResultSize(); + } else { + return false; + } } // Used for job.query API endpoint From 8e683e7a93f8505242cb4b3cf2fb40d2ddd911ae Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 29 Dec 2021 14:58:22 +0530 Subject: [PATCH 090/277] Added BQResultSet Test cases --- .../cloud/bigquery/it/ITBigQueryTest.java | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 4de7efffc..83693f18d 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -2221,6 +2221,80 @@ public void testQueryExternalHivePartitioningOptionCustomLayout() throws Interru assertTrue(bigquery.delete(tableId)); } + @Test + public void testConnectionImplDryRun() throws SQLException { + String query = + "select StringField, BigNumericField, BooleanField, BytesField, IntegerField, TimestampField, FloatField, " + + "NumericField, TimeField, DateField, DateTimeField , GeographyField, RecordField.BytesField, RecordField.BooleanField, IntegerArrayField from " + + TABLE_ID_FASTQUERY_BQ_RESULTSET.getTable() + + " order by TimestampField"; + ConnectionSettings connectionSettings = + ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); + Connection connection = bigquery.createConnection(connectionSettings); + BigQueryDryRunResult bigQueryDryRunResultSet = connection.dryRun(query); + Schema sc = bigQueryDryRunResultSet.getSchema(); + assertEquals(BQ_RESULTSET_EXPECTED_SCHEMA, sc); // match the schema + // TODO(prasmish): Validate QueryParameters after the implementation is complete + } + + @Test + // This test case test the order of the records, making sure that the result is not jumbled up due + // to the multithreaded BigQueryResultSet implementation + public void testBQResultSetMultiThreadedOrder() throws SQLException { + String query = + "SELECT date FROM " + + TABLE_ID_LARGE.getTable() + + " where date is not null order by date asc limit 300000"; + ConnectionSettings connectionSettings = + ConnectionSettings.newBuilder() + .setDefaultDataset(DatasetId.of(DATASET)) + .setNumBufferedRows(10000L) // page size + .build(); + Connection connection = bigquery.createConnection(connectionSettings); + BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); + ResultSet rs = bigQueryResultSet.getResultSet(); + int cnt = 0; + assertTrue(rs.next()); + ++cnt; + java.sql.Date lastDate = rs.getDate(0); + while (rs.next()) { + assertNotNull(rs.getDate(0)); + assertTrue(rs.getDate(0).getTime() >= lastDate.getTime()); // sorted order is maintained + lastDate = rs.getDate(0); + ++cnt; + } + assertEquals(300000, cnt); // total 300000 rows should be read + } + + @Test + public void testBQResultSetPaginationSlowQuery() throws SQLException { + String query = + "SELECT date, county, state_name, confirmed_cases, deaths FROM " + + TABLE_ID_LARGE.getTable() + + " where date is not null and county is not null and state_name is not null order by date limit 300000"; + ConnectionSettings connectionSettings = + ConnectionSettings.newBuilder() + .setDefaultDataset(DatasetId.of(DATASET)) + .setNumBufferedRows(10000L) // page size + .setJobTimeoutMs( + 15000L) // So that ConnectionImpl.isFastQuerySupported returns false, and the slow + // query route gets executed + .build(); + Connection connection = bigquery.createConnection(connectionSettings); + BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); + ResultSet rs = bigQueryResultSet.getResultSet(); + int cnt = 0; + while (rs.next()) { // pagination starts after approx 120,000 records + assertNotNull(rs.getDate(0)); + assertNotNull(rs.getString(1)); + assertNotNull(rs.getString(2)); + assertTrue(rs.getInt(3) >= 0); + assertTrue(rs.getInt(4) >= 0); + ++cnt; + } + assertEquals(300000, cnt); // total 300000 rows should be read + } + @Test public void testExecuteSelectSinglePageTableRow() throws SQLException { String query = From 0e8b973974caf346eeb3642480ae869433c8c5f7 Mon Sep 17 00:00:00 2001 From: stephwang Date: Wed, 29 Dec 2021 11:03:06 -0500 Subject: [PATCH 091/277] code cleanup and refactoring --- .../google/cloud/bigquery/ConnectionImpl.java | 267 +++++++----------- 1 file changed, 108 insertions(+), 159 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 87ab7a733..da30ddaac 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -40,14 +40,13 @@ /** Implementation for {@link Connection}, the generic BigQuery connection API (not JDBC). */ final class ConnectionImpl implements Connection { - private ConnectionSettings connectionSettings; - private BigQueryOptions bigQueryOptions; - private BigQueryRpc bigQueryRpc; - private BigQueryRetryConfig retryConfig; - // false even after interrupting it here! This could be due to the retrial logic, so - // using it as a workaround. + private final ConnectionSettings connectionSettings; + private final BigQueryOptions bigQueryOptions; + private final BigQueryRpc bigQueryRpc; + private final BigQueryRetryConfig retryConfig; + private final int bufferSize; // buffer size in Producer Thread private final int MAX_PROCESS_QUERY_THREADS_CNT = 5; - private ExecutorService queryTaskExecutor = + private final ExecutorService queryTaskExecutor = Executors.newFixedThreadPool(MAX_PROCESS_QUERY_THREADS_CNT); private final Logger logger = Logger.getLogger(this.getClass().getName()); @@ -60,14 +59,18 @@ final class ConnectionImpl implements Connection { this.bigQueryOptions = bigQueryOptions; this.bigQueryRpc = bigQueryRpc; this.retryConfig = retryConfig; + this.bufferSize = + (int) + (connectionSettings.getNumBufferedRows() == null + || connectionSettings.getNumBufferedRows() < 10000 + ? 20000 + : Math.min(connectionSettings.getNumBufferedRows() * 2, 100000)); } /* - Cancel method shutdowns the pageFetcher and producerWorker threads gracefully using interrupt - - The pageFetcher threat wont request for any subsequent threads after interrupting and shutdown as soon as any ongoing RPC call returns - - The producerWorker wont populate the buffer with any further records and clear the buffer, put a EoF marker and shutdown + Cancel method shutdowns the pageFetcher and producerWorker threads gracefully using interrupt. + The pageFetcher threat will not request for any subsequent threads after interrupting and shutdown as soon as any ongoing RPC call returns. + The producerWorker will not populate the buffer with any further records and clear the buffer, put a EoF marker and shutdown. */ @Override public synchronized Boolean cancel() throws BigQuerySQLException { @@ -134,8 +137,8 @@ public BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException { createQueryJob(sql, connectionSettings, null, null); JobId jobId = JobId.fromPb(queryJob.getJobReference()); GetQueryResultsResponse firstPage = getQueryResultsFirstPage(jobId); - return getQueryResultsWithJobId( - firstPage.getTotalRows().longValue(), firstPage.getRows().size(), null, jobId, firstPage); + return getQueryResultsWithJob( + firstPage.getTotalRows().longValue(), firstPage.getRows().size(), jobId, firstPage); } @Override @@ -154,8 +157,8 @@ public BigQueryResultSet executeSelect( createQueryJob(sql, connectionSettings, parameters, labels); JobId jobId = JobId.fromPb(queryJob.getJobReference()); GetQueryResultsResponse firstPage = getQueryResultsFirstPage(jobId); - return getQueryResultsWithJobId( - firstPage.getTotalRows().longValue(), firstPage.getRows().size(), null, jobId, firstPage); + return getQueryResultsWithJob( + firstPage.getTotalRows().longValue(), firstPage.getRows().size(), jobId, firstPage); } static class EndOfFieldValueList @@ -192,8 +195,8 @@ private BigQueryResultSet queryRpc(final String projectId, final QueryRequest qu results.getErrors().stream() .map(BigQueryError.FROM_PB_FUNCTION) .collect(Collectors.toList()); - // Throwing BigQueryException since there may be no JobId and we want to stay consistent - // with the case where there there is a HTTP error + // Throwing BigQueryException since there may be no JobId, and we want to stay consistent + // with the case where there is an HTTP error throw new BigQueryException(bigQueryErrors); } @@ -201,72 +204,27 @@ private BigQueryResultSet queryRpc(final String projectId, final QueryRequest qu if (results.getJobComplete() && results.getSchema() != null) { return processQueryResponseResults(results); } else { - // Query is long running (> 10s) and hasn't completed yet, or query completed but didn't + // Query is long-running (> 10s) and hasn't completed yet, or query completed but didn't // return the schema, fallback to jobs.insert path. Some operations don't return the schema - // and - // can be optimized here, but this is left as future work. + // and can be optimized here, but this is left as future work. long totalRows = results.getTotalRows().longValue(); long pageRows = results.getRows().size(); JobId jobId = JobId.fromPb(results.getJobReference()); - return getQueryResultsWithJobId(totalRows, pageRows, null, jobId, null); - } - } - - private static Iterable getIterableFieldValueList( - Iterable tableDataPb, final Schema schema) { - return ImmutableList.copyOf( - Iterables.transform( - tableDataPb != null ? tableDataPb : ImmutableList.of(), - new Function() { - FieldList fields = schema != null ? schema.getFields() : null; - - @Override - public FieldValueList apply(TableRow rowPb) { - return FieldValueList.fromPb(rowPb.getF(), fields); - } - })); - } - - // Determines the optimal number of caches pages to improve read performance - private int getPageCacheSize(Long numBufferedRows, long numRows, Schema schema) { - final int MIN_CACHE_SIZE = 3; // Min number of pages in the page size - final int MAX_CACHE_SIZE = 20; // //Min number of pages in the page size - int columnsRead = schema.getFields().size(); - int numCachedPages; - long numCachedRows = numBufferedRows == null ? 0 : numBufferedRows.longValue(); - - // TODO: Further enhance this logic - if (numCachedRows > 10000) { - numCachedPages = - 2; // the size of numBufferedRows is quite large and as per our tests we should be able to - // do enough even with low - } else if (columnsRead > 15 - && numCachedRows - > 5000) { // too many fields are being read, setting the page size on the lower end - numCachedPages = 3; - } else if (numCachedRows < 2000 - && columnsRead < 15) { // low pagesize with fewer number of columns, we can cache more pages - numCachedPages = 20; - } else { // default - under 10K numCachedRows with any number of columns - numCachedPages = 5; + GetQueryResultsResponse firstPage = getQueryResultsFirstPage(jobId); + return getQueryResultsWithJob(totalRows, pageRows, jobId, firstPage); } - return numCachedPages < MIN_CACHE_SIZE - ? MIN_CACHE_SIZE - : (Math.min( - numCachedPages, - MAX_CACHE_SIZE)); // numCachedPages should be between the defined min and max } // TODO: Write a IT with order by query /* This method processed the first page of GetQueryResultsResponse and then it uses tabledata.list */ - private BigQueryResultSet processGetQueryResponseResults( - GetQueryResultsResponse results, JobId completeJobId) { + private BigQueryResultSet getSubsequentPagesOfResults( + GetQueryResultsResponse firstPage, JobId completeJobId) { Schema schema; long numRows; - schema = Schema.fromPb(results.getSchema()); - numRows = results.getTotalRows().longValue(); + schema = Schema.fromPb(firstPage.getSchema()); + numRows = firstPage.getTotalRows().longValue(); - // Get query statistics + // Create GetQueryResultsResponse query statistics Job queryJob = getQueryJobRpc(completeJobId); JobStatistics.QueryStatistics statistics = queryJob.getStatistics(); DmlStats dmlStats = statistics.getDmlStats() == null ? null : statistics.getDmlStats(); @@ -275,39 +233,25 @@ private BigQueryResultSet processGetQueryResponseResults( BigQueryResultSetStats bigQueryResultSetStats = new BigQueryResultSetStatsImpl(dmlStats, sessionInfo); - // Producer thread for populating the buffer row by row - // Keeping the buffersize more than the page size and ensuring it's always a reasonable number - int bufferSize = - (int) - (connectionSettings.getNumBufferedRows() == null - || connectionSettings.getNumBufferedRows() < 10000 - ? 20000 - : Math.min( - connectionSettings.getNumBufferedRows() * 2, - 100000)); // ensuring that the buffer has values between 20K and 100K - BlockingQueue> buffer = - new LinkedBlockingDeque<>( - bufferSize); // this keeps the deserialized records at the row level, which will be - // consumed by the BQResultSet + // Keeps the deserialized records at the row level, which is consumed by BigQueryResultSet + BlockingQueue> buffer = new LinkedBlockingDeque<>(bufferSize); + + // Keeps the parsed FieldValueLists BlockingQueue, Boolean>> pageCache = new LinkedBlockingDeque<>( - getPageCacheSize( - connectionSettings.getNumBufferedRows(), - numRows, - schema)); // this keeps the parsed FieldValueLists + getPageCacheSize(connectionSettings.getNumBufferedRows(), numRows, schema)); + + // Keeps the raw RPC responses BlockingQueue> rpcResponseQueue = new LinkedBlockingDeque<>( - getPageCacheSize( - connectionSettings.getNumBufferedRows(), - numRows, - schema)); // this keeps the raw RPC response + getPageCacheSize(connectionSettings.getNumBufferedRows(), numRows, schema)); - JobId jobId = JobId.fromPb(results.getJobReference()); + JobId jobId = JobId.fromPb(firstPage.getJobReference()); - runNextPageTaskAsync(results.getPageToken(), queryJobsGetRpc(jobId), rpcResponseQueue); + runNextPageTaskAsync(firstPage.getPageToken(), getDestinationTable(jobId), rpcResponseQueue); parseRpcDataAsync( - results.getRows(), + firstPage.getRows(), schema, pageCache, rpcResponseQueue); // parses data on a separate thread, thus maximising processing @@ -328,7 +272,7 @@ private BigQueryResultSet processQueryResponseResults( schema = Schema.fromPb(results.getSchema()); numRows = results.getTotalRows().longValue(); - // Get query statistics + // Create QueryResponse query statistics DmlStats dmlStats = results.getDmlStats() == null ? null : DmlStats.fromPb(results.getDmlStats()); JobStatistics.SessionInfo sessionInfo = @@ -338,47 +282,21 @@ private BigQueryResultSet processQueryResponseResults( BigQueryResultSetStats bigQueryResultSetStats = new BigQueryResultSetStatsImpl(dmlStats, sessionInfo); - // Producer thread for populating the buffer row by row - // Keeping the buffersize more than the page size and ensuring it's always a reasonable number - int bufferSize = - (int) - (connectionSettings.getNumBufferedRows() == null - || connectionSettings.getNumBufferedRows() < 10000 - ? 20000 - : Math.min( - connectionSettings.getNumBufferedRows() * 2, - 100000)); // ensuring that the buffer has values between 20K and 100K - BlockingQueue> buffer = - new LinkedBlockingDeque<>( - bufferSize); // this keeps the deserialized records at the row level, which will be - // consumed by the BQResultSet + BlockingQueue> buffer = new LinkedBlockingDeque<>(bufferSize); BlockingQueue, Boolean>> pageCache = new LinkedBlockingDeque<>( - getPageCacheSize( - connectionSettings.getNumBufferedRows(), - numRows, - schema)); // this keeps the parsed FieldValueLists + getPageCacheSize(connectionSettings.getNumBufferedRows(), numRows, schema)); BlockingQueue> rpcResponseQueue = new LinkedBlockingDeque<>( - getPageCacheSize( - connectionSettings.getNumBufferedRows(), - numRows, - schema)); // this keeps the raw RPC response + getPageCacheSize(connectionSettings.getNumBufferedRows(), numRows, schema)); JobId jobId = JobId.fromPb(results.getJobReference()); - runNextPageTaskAsync(results.getPageToken(), queryJobsGetRpc(jobId), rpcResponseQueue); + runNextPageTaskAsync(results.getPageToken(), getDestinationTable(jobId), rpcResponseQueue); - parseRpcDataAsync( - results.getRows(), - schema, - pageCache, - rpcResponseQueue); // parses data on a separate thread, thus maximising processing - // throughput + parseRpcDataAsync(results.getRows(), schema, pageCache, rpcResponseQueue); - populateBufferAsync( - rpcResponseQueue, pageCache, buffer); // spawns a thread to populate the buffer + populateBufferAsync(rpcResponseQueue, pageCache, buffer); - // This will work for pagination as well, as buffer is getting updated asynchronously return new BigQueryResultSetImpl>( schema, numRows, buffer, bigQueryResultSetStats); } @@ -533,17 +451,59 @@ private void populateBufferAsync( queryTaskExecutor.execute(populateBufferRunnable); } + /* Helper method that parse and populate a page with TableRows */ + private static Iterable getIterableFieldValueList( + Iterable tableDataPb, final Schema schema) { + return ImmutableList.copyOf( + Iterables.transform( + tableDataPb != null ? tableDataPb : ImmutableList.of(), + new Function() { + final FieldList fields = schema != null ? schema.getFields() : null; + + @Override + public FieldValueList apply(TableRow rowPb) { + return FieldValueList.fromPb(rowPb.getF(), fields); + } + })); + } + + /* Helper method that determines the optimal number of caches pages to improve read performance */ + private int getPageCacheSize(Long numBufferedRows, long numRows, Schema schema) { + final int MIN_CACHE_SIZE = 3; // Min number of pages in the page size + final int MAX_CACHE_SIZE = 20; // //Min number of pages in the page size + int columnsRead = schema.getFields().size(); + int numCachedPages; + long numCachedRows = numBufferedRows == null ? 0 : numBufferedRows.longValue(); + + // TODO: Further enhance this logic + if (numCachedRows > 10000) { + numCachedPages = + 2; // the size of numBufferedRows is quite large and as per our tests we should be able to + // do enough even with low + } else if (columnsRead > 15 + && numCachedRows + > 5000) { // too many fields are being read, setting the page size on the lower end + numCachedPages = 3; + } else if (numCachedRows < 2000 + && columnsRead < 15) { // low pagesize with fewer number of columns, we can cache more pages + numCachedPages = 20; + } else { // default - under 10K numCachedRows with any number of columns + numCachedPages = 5; + } + return numCachedPages < MIN_CACHE_SIZE + ? MIN_CACHE_SIZE + : (Math.min( + numCachedPages, + MAX_CACHE_SIZE)); // numCachedPages should be between the defined min and max + } + /* Returns query results using either tabledata.list or the high throughput Read API */ - private BigQueryResultSet getQueryResultsWithJobId( - long totalRows, - long pageRows, - Schema schema, - JobId jobId, - GetQueryResultsResponse firstPage) { - TableId destinationTable = queryJobsGetRpc(jobId); + private BigQueryResultSet getQueryResultsWithJob( + long totalRows, long pageRows, JobId jobId, GetQueryResultsResponse firstPage) { + TableId destinationTable = getDestinationTable(jobId); return useReadAPI(totalRows, pageRows) ? highThroughPutRead(destinationTable) - : getQueryResultsRpc(jobId, firstPage); + : getSubsequentPagesOfResults(firstPage, jobId); } /* Returns Job from jobId by calling the jobs.get API */ @@ -577,7 +537,7 @@ private Job getQueryJobRpc(JobId jobId) { } /* Returns the destinationTable from jobId by calling jobs.get API */ - private TableId queryJobsGetRpc(JobId jobId) { + private TableId getDestinationTable(JobId jobId) { Job job = getQueryJobRpc(jobId); return ((QueryJobConfiguration) job.getConfiguration()).getDestinationTable(); } @@ -614,20 +574,6 @@ private BigQueryResultSet highThroughPutRead(TableId destinationTable) { return null; } - /* Returns results of the query associated with the provided job using jobs.getQueryResults API */ - private BigQueryResultSet getQueryResultsRpc(JobId jobId, GetQueryResultsResponse firstPage) { - try { - // paginate using multi threaded logic and return BigQueryResultSet - if (firstPage == null) { - return processGetQueryResponseResults(getQueryResultsFirstPage(jobId), jobId); - } else { // we already received the firstpage using the jobId - return processGetQueryResponseResults(firstPage, jobId); - } - } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) { - throw BigQueryException.translateAndThrow(e); - } - } - /*Returns just the first page of GetQueryResultsResponse using the jobId*/ private GetQueryResultsResponse getQueryResultsFirstPage(JobId jobId) { JobId completeJobId = @@ -730,12 +676,15 @@ private boolean isFastQuerySupported() { } private boolean useReadAPI(Long totalRows, Long pageRows) { - long resultRatio = totalRows / pageRows; - return resultRatio - > connectionSettings - .getReadClientConnectionConfiguration() - .getTotalToPageRowCountRatio() - && totalRows > connectionSettings.getReadClientConnectionConfiguration().getMinResultSize(); + return false; + // TODO: IMPLEMENT READ API LOGIC IN THE NEXT PHASE + // long resultRatio = totalRows / pageRows; + // return resultRatio + // > connectionSettings + // .getReadClientConnectionConfiguration() + // .getTotalToPageRowCountRatio() + // && totalRows > + // connectionSettings.getReadClientConnectionConfiguration().getMinResultSize(); } // Used for job.query API endpoint From 8d65193a8bf27b81fdc0012be939a4307772e9d7 Mon Sep 17 00:00:00 2001 From: stephwang Date: Wed, 29 Dec 2021 11:11:33 -0500 Subject: [PATCH 092/277] add import --- .../test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 83693f18d..862e11508 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -48,6 +48,7 @@ import com.google.cloud.bigquery.BigQuery.JobOption; import com.google.cloud.bigquery.BigQuery.TableField; import com.google.cloud.bigquery.BigQuery.TableOption; +import com.google.cloud.bigquery.BigQueryDryRunResult; import com.google.cloud.bigquery.BigQueryError; import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.BigQueryResultSet; From e8f1e71e6b19cf7ca6e25c8153426791048a6e02 Mon Sep 17 00:00:00 2001 From: stephwang Date: Wed, 29 Dec 2021 11:37:42 -0500 Subject: [PATCH 093/277] additional lint --- .../google/cloud/bigquery/ConnectionImpl.java | 22 +++++++++---------- .../cloud/bigquery/it/ITBigQueryTest.java | 4 ++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index fe50934e6..4bfa60ac2 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -137,7 +137,7 @@ public BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException { createQueryJob(sql, connectionSettings, null, null); JobId jobId = JobId.fromPb(queryJob.getJobReference()); GetQueryResultsResponse firstPage = getQueryResultsFirstPage(jobId); - return getQueryResultsWithJob( + return getSubsequentQueryResultsWithJob( firstPage.getTotalRows().longValue(), firstPage.getRows().size(), jobId, firstPage); } @@ -157,7 +157,7 @@ public BigQueryResultSet executeSelect( createQueryJob(sql, connectionSettings, parameters, labels); JobId jobId = JobId.fromPb(queryJob.getJobReference()); GetQueryResultsResponse firstPage = getQueryResultsFirstPage(jobId); - return getQueryResultsWithJob( + return getSubsequentQueryResultsWithJob( firstPage.getTotalRows().longValue(), firstPage.getRows().size(), jobId, firstPage); } @@ -211,21 +211,20 @@ private BigQueryResultSet queryRpc(final String projectId, final QueryRequest qu long pageRows = results.getRows().size(); JobId jobId = JobId.fromPb(results.getJobReference()); GetQueryResultsResponse firstPage = getQueryResultsFirstPage(jobId); - return getQueryResultsWithJob(totalRows, pageRows, jobId, firstPage); + return getSubsequentQueryResultsWithJob(totalRows, pageRows, jobId, firstPage); } } // TODO: Write a IT with order by query /* This method processed the first page of GetQueryResultsResponse and then it uses tabledata.list */ - private BigQueryResultSet getSubsequentPagesOfResults( - GetQueryResultsResponse firstPage, JobId completeJobId) { + private BigQueryResultSet tableDataList(GetQueryResultsResponse firstPage, JobId jobId) { Schema schema; long numRows; schema = Schema.fromPb(firstPage.getSchema()); numRows = firstPage.getTotalRows().longValue(); // Create GetQueryResultsResponse query statistics - Job queryJob = getQueryJobRpc(completeJobId); + Job queryJob = getQueryJobRpc(jobId); JobStatistics.QueryStatistics statistics = queryJob.getStatistics(); DmlStats dmlStats = statistics.getDmlStats() == null ? null : statistics.getDmlStats(); JobStatistics.SessionInfo sessionInfo = @@ -246,8 +245,6 @@ private BigQueryResultSet getSubsequentPagesOfResults( new LinkedBlockingDeque<>( getPageCacheSize(connectionSettings.getNumBufferedRows(), numRows, schema)); - JobId jobId = JobId.fromPb(firstPage.getJobReference()); - runNextPageTaskAsync(firstPage.getPageToken(), getDestinationTable(jobId), rpcResponseQueue); parseRpcDataAsync( @@ -291,6 +288,7 @@ private BigQueryResultSet processQueryResponseResults( getPageCacheSize(connectionSettings.getNumBufferedRows(), numRows, schema)); JobId jobId = JobId.fromPb(results.getJobReference()); + runNextPageTaskAsync(results.getPageToken(), getDestinationTable(jobId), rpcResponseQueue); parseRpcDataAsync(results.getRows(), schema, pageCache, rpcResponseQueue); @@ -498,12 +496,14 @@ private int getPageCacheSize(Long numBufferedRows, long numRows, Schema schema) } /* Returns query results using either tabledata.list or the high throughput Read API */ - private BigQueryResultSet getQueryResultsWithJob( + private BigQueryResultSet getSubsequentQueryResultsWithJob( long totalRows, long pageRows, JobId jobId, GetQueryResultsResponse firstPage) { TableId destinationTable = getDestinationTable(jobId); return useReadAPI(totalRows, pageRows) - ? highThroughPutRead(destinationTable) - : getSubsequentPagesOfResults(firstPage, jobId); + ? highThroughPutRead( + destinationTable) // discord first page and stream the entire BigQueryResultSet using + // the Read API + : tableDataList(firstPage, jobId); } /* Returns Job from jobId by calling the jobs.get API */ diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 862e11508..041a61272 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -2233,8 +2233,8 @@ public void testConnectionImplDryRun() throws SQLException { ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); Connection connection = bigquery.createConnection(connectionSettings); BigQueryDryRunResult bigQueryDryRunResultSet = connection.dryRun(query); - Schema sc = bigQueryDryRunResultSet.getSchema(); - assertEquals(BQ_RESULTSET_EXPECTED_SCHEMA, sc); // match the schema + // Schema sc = bigQueryDryRunResultSet.getSchema(); + // assertEquals(BQ_RESULTSET_EXPECTED_SCHEMA, sc); // match the schema // TODO(prasmish): Validate QueryParameters after the implementation is complete } From 26bc6bb78fbf7f502b5fe07effb24defbc6dd1e6 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Wed, 29 Dec 2021 16:40:07 +0000 Subject: [PATCH 094/277] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .../src/main/java/com/google/cloud/bigquery/ConnectionImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 4bfa60ac2..3cd89bf1e 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -502,7 +502,7 @@ private BigQueryResultSet getSubsequentQueryResultsWithJob( return useReadAPI(totalRows, pageRows) ? highThroughPutRead( destinationTable) // discord first page and stream the entire BigQueryResultSet using - // the Read API + // the Read API : tableDataList(firstPage, jobId); } From 4aea35a261b1fa2b8c1f0a3467b9f88f39f4fa5d Mon Sep 17 00:00:00 2001 From: stephwang Date: Wed, 29 Dec 2021 11:54:32 -0500 Subject: [PATCH 095/277] add test requirements --- .../google/cloud/bigquery/ConnectionImpl.java | 2 +- .../cloud/bigquery/ConnectionImplTest.java | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 4bfa60ac2..3cd89bf1e 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -502,7 +502,7 @@ private BigQueryResultSet getSubsequentQueryResultsWithJob( return useReadAPI(totalRows, pageRows) ? highThroughPutRead( destinationTable) // discord first page and stream the entire BigQueryResultSet using - // the Read API + // the Read API : tableDataList(firstPage, jobId); } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java new file mode 100644 index 000000000..0004c4ae0 --- /dev/null +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java @@ -0,0 +1,45 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +public class ConnectionImplTest { + + // TODO: Add testNextPageTask() + + // TODO: Add testParseDataTask() + + // TODO: Add testPopulateBuffer() + + // TODO: Add testGetQueryResultsFirstPage() + + // TODO: Add testFastQuerySinglePage() + + // TODO: Add testFastQueryMultiplePages() --> should exercise processQueryResponseResults() method + + // TODO: Add testFastQueryLongRunning() --> should exercise getSubsequentQueryResultsWithJob() + // method + + // TODO: Add testLegacyQuerySinglePage() --> should exercise createQueryJob() and + // getQueryResultsFirstPage() + + // TODO: Add testLegacyQueryMultiplePages() --> should exercise processQueryResponseResults() + // method + + // TODO: Add testQueryDryRun() + + // TODO: Add testCancel() +} From a9c47ef297d75f7b21a6a72d249a60f6e9bbe191 Mon Sep 17 00:00:00 2001 From: stephwang Date: Sat, 8 Jan 2022 20:41:25 -0500 Subject: [PATCH 096/277] add dryRun implementation --- .../google/cloud/bigquery/ConnectionImpl.java | 95 ++++++++++--------- .../google/cloud/bigquery/JobStatistics.java | 26 ++++- .../cloud/bigquery/QueryJobConfiguration.java | 5 + .../cloud/bigquery/it/ITBigQueryTest.java | 16 ++-- 4 files changed, 89 insertions(+), 53 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 3cd89bf1e..644b565eb 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -19,7 +19,12 @@ import static com.google.cloud.RetryHelper.runWithRetries; import static java.net.HttpURLConnection.HTTP_NOT_FOUND; -import com.google.api.services.bigquery.model.*; +import com.google.api.services.bigquery.model.GetQueryResultsResponse; +import com.google.api.services.bigquery.model.JobConfigurationQuery; +import com.google.api.services.bigquery.model.QueryParameter; +import com.google.api.services.bigquery.model.QueryRequest; +import com.google.api.services.bigquery.model.TableDataList; +import com.google.api.services.bigquery.model.TableRow; import com.google.cloud.RetryHelper; import com.google.cloud.Tuple; import com.google.cloud.bigquery.spi.v2.BigQueryRpc; @@ -28,7 +33,10 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; -import java.util.*; +import java.util.AbstractList; +import java.util.List; +import java.util.Map; +import java.util.UUID; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -80,48 +88,11 @@ public synchronized Boolean cancel() throws BigQuerySQLException { @Override public BigQueryDryRunResult dryRun(String sql) throws BigQuerySQLException { - QueryJobConfiguration queryConfig = - QueryJobConfiguration.newBuilder(sql).setDryRun(true).build(); - // Job job = BigQueryOptions.getDefaultInstance().getService().create(JobInfo.of(queryConfig)); - /*//TODO: WIP implementation - JobId jobId = JobId.of(UUID.randomUUID().toString()); - Job job = BigQueryOptions.getDefaultInstance().getService().create(JobInfo.newBuilder(queryConfig).setJobId(jobId).build()); - - try { - if(!job.isDone()){//wait if job is still running. TODO(prasmish): Doublecheck if this check is required - job = job.waitFor(); - } - - // response.get - } catch (InterruptedException e) { - e.printStackTrace(); - } - JobStatistics.QueryStatistics statistics = job.getStatistics(); - try { - TableResult tr = job.getQueryResults(); - // job.getBigQuery().getQueryResults(). - // tr.get - // job.getConfiguration().g - } catch (InterruptedException e) { - e.printStackTrace(); - } - com.google.cloud.bigquery.QueryResponse response = BigQueryOptions.getDefaultInstance().getService().getQueryResults(jobId); - JobId jobId = JobId.fromPb(response.getJobReference()); - - com.google.api.services.bigquery.model.QueryResponse results; - results.getJobReference().getJobId() - - JobId jobIdd = JobId.fromPb(results.getJobReference()).get; - //response. - - JobConfigurationQuery d = queryConfig.toPb().getQuery(); - List qp = d.getQueryParameters(); - //response. - return new BigQueryDryRunResultImpl( - response.getSchema(), queryConfig.toPb().getQuery().getQueryParameters()); - - */ - return null; + com.google.api.services.bigquery.model.Job dryRunJob = createDryRunJob(sql); + Schema schema = Schema.fromPb(dryRunJob.getStatistics().getQuery().getSchema()); + List queryParameters = + dryRunJob.getStatistics().getQuery().getUndeclaredQueryParameters(); + return new BigQueryDryRunResultImpl(schema, queryParameters); } @Override @@ -730,7 +701,7 @@ private QueryRequest createQueryRequest( return content; } - // Used for jobs.getQueryResults API endpoint + // Used by jobs.getQueryResults API endpoint private com.google.api.services.bigquery.model.Job createQueryJob( String sql, ConnectionSettings connectionSettings, @@ -847,4 +818,38 @@ private com.google.api.services.bigquery.model.Job createQueryJob( } return queryJob; } + + // Used by dryRun + private com.google.api.services.bigquery.model.Job createDryRunJob(String sql) { + com.google.api.services.bigquery.model.JobConfiguration configurationPb = + new com.google.api.services.bigquery.model.JobConfiguration(); + configurationPb.setDryRun(true); + JobConfigurationQuery queryConfigurationPb = new JobConfigurationQuery(); + String parameterMode = sql.contains("?") ? "POSITIONAL" : "NAMED"; + queryConfigurationPb.setParameterMode(parameterMode); + queryConfigurationPb.setQuery(sql); + // UndeclaredQueryParameter is only supported in StandardSQL + queryConfigurationPb.setUseLegacySql(false); + if (connectionSettings.getDefaultDataset() != null) { + queryConfigurationPb.setDefaultDataset(connectionSettings.getDefaultDataset().toPb()); + } + configurationPb.setQuery(queryConfigurationPb); + + com.google.api.services.bigquery.model.Job jobPb = + JobInfo.of(QueryJobConfiguration.fromPb(configurationPb)).toPb(); + + com.google.api.services.bigquery.model.Job dryRunJob; + try { + dryRunJob = + BigQueryRetryHelper.runWithRetries( + () -> bigQueryRpc.createJobForQuery(jobPb), + bigQueryOptions.getRetrySettings(), + BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, + bigQueryOptions.getClock(), + retryConfig); + } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) { + throw BigQueryException.translateAndThrow(e); + } + return dryRunJob; + } } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java index ab9fdabb3..0ef1d1f94 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java @@ -21,6 +21,7 @@ import com.google.api.services.bigquery.model.JobStatistics2; import com.google.api.services.bigquery.model.JobStatistics3; import com.google.api.services.bigquery.model.JobStatistics4; +import com.google.api.services.bigquery.model.QueryParameter; import com.google.cloud.StringEnumType; import com.google.cloud.StringEnumValue; import com.google.common.base.Function; @@ -339,6 +340,7 @@ public static class QueryStatistics extends JobStatistics { private final List queryPlan; private final List timeline; private final Schema schema; + private final List queryParameters; /** * StatementType represents possible types of SQL statements reported as part of the @@ -421,6 +423,7 @@ static final class Builder extends JobStatistics.Builder queryPlan; private List timeline; private Schema schema; + private List queryParameters; private Builder() {} @@ -569,6 +572,11 @@ Builder setSchema(Schema schema) { return self(); } + Builder setQueryParameters(List queryParameters) { + this.queryParameters = queryParameters; + return self(); + } + @Override QueryStatistics build() { return new QueryStatistics(this); @@ -595,6 +603,7 @@ private QueryStatistics(Builder builder) { this.queryPlan = builder.queryPlan; this.timeline = builder.timeline; this.schema = builder.schema; + this.queryParameters = builder.queryParameters; } /** Returns query statistics specific to the use of BI Engine. */ @@ -715,6 +724,14 @@ public Schema getSchema() { return schema; } + /** + * Standard SQL only: Returns a list of undeclared query parameters detected during a dry run + * validation. + */ + public List getQueryParameters() { + return queryParameters; + } + @Override ToStringHelper toStringHelper() { return super.toStringHelper() @@ -725,7 +742,8 @@ ToStringHelper toStringHelper() { .add("totalBytesProcessed", totalBytesProcessed) .add("queryPlan", queryPlan) .add("timeline", timeline) - .add("schema", schema); + .add("schema", schema) + .add("queryParameters", queryParameters); } @Override @@ -746,7 +764,8 @@ public final int hashCode() { totalBytesBilled, totalBytesProcessed, queryPlan, - schema); + schema, + queryParameters); } @Override @@ -788,6 +807,9 @@ com.google.api.services.bigquery.model.JobStatistics toPb() { if (schema != null) { queryStatisticsPb.setSchema(schema.toPb()); } + if (queryParameters != null) { + queryStatisticsPb.setUndeclaredQueryParameters(queryParameters); + } return super.toPb().setQuery(queryStatisticsPb); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryJobConfiguration.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryJobConfiguration.java index 48ec22caf..cc726bdd1 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryJobConfiguration.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryJobConfiguration.java @@ -166,6 +166,11 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur this(); JobConfigurationQuery queryConfigurationPb = configurationPb.getQuery(); this.query = queryConfigurationPb.getQuery(); + // Allows to get undeclaredqueryparameters in jobstatistics2 + if (queryConfigurationPb.getQueryParameters() == null + && queryConfigurationPb.getParameterMode() != null) { + parameterMode = queryConfigurationPb.getParameterMode(); + } if (queryConfigurationPb.getQueryParameters() != null && !queryConfigurationPb.getQueryParameters().isEmpty()) { if (queryConfigurationPb.getQueryParameters().get(0).getName() == null) { diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 041a61272..95ecad668 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -29,6 +29,8 @@ import static org.junit.Assert.fail; import com.google.api.gax.paging.Page; +import com.google.api.services.bigquery.model.QueryParameter; +import com.google.api.services.bigquery.model.QueryParameterType; import com.google.auth.oauth2.GoogleCredentials; import com.google.auth.oauth2.ServiceAccountCredentials; import com.google.cloud.Date; @@ -2225,17 +2227,19 @@ public void testQueryExternalHivePartitioningOptionCustomLayout() throws Interru @Test public void testConnectionImplDryRun() throws SQLException { String query = - "select StringField, BigNumericField, BooleanField, BytesField, IntegerField, TimestampField, FloatField, " - + "NumericField, TimeField, DateField, DateTimeField , GeographyField, RecordField.BytesField, RecordField.BooleanField, IntegerArrayField from " + "select StringField from " + TABLE_ID_FASTQUERY_BQ_RESULTSET.getTable() - + " order by TimestampField"; + + " where StringField = ?"; ConnectionSettings connectionSettings = ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); Connection connection = bigquery.createConnection(connectionSettings); BigQueryDryRunResult bigQueryDryRunResultSet = connection.dryRun(query); - // Schema sc = bigQueryDryRunResultSet.getSchema(); - // assertEquals(BQ_RESULTSET_EXPECTED_SCHEMA, sc); // match the schema - // TODO(prasmish): Validate QueryParameters after the implementation is complete + assertNotNull(bigQueryDryRunResultSet.getSchema()); + List queryParameters = bigQueryDryRunResultSet.getQueryParameters(); + List expectedQueryParameters = + ImmutableList.of( + new QueryParameter().setParameterType(new QueryParameterType().setType("STRING"))); + assertEquals(expectedQueryParameters, queryParameters); } @Test From b738b574763b98cec1eba754d455559832db05f2 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Mon, 17 Jan 2022 17:03:36 +0530 Subject: [PATCH 097/277] Added schema check @ testConnectionImplDryRun --- .../java/com/google/cloud/bigquery/it/ITBigQueryTest.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 95ecad668..badb500f4 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -2227,14 +2227,16 @@ public void testQueryExternalHivePartitioningOptionCustomLayout() throws Interru @Test public void testConnectionImplDryRun() throws SQLException { String query = - "select StringField from " - + TABLE_ID_FASTQUERY_BQ_RESULTSET.getTable() - + " where StringField = ?"; + String.format( + "select StringField, BigNumericField, BooleanField, BytesField, IntegerField, TimestampField, FloatField, NumericField, TimeField, DateField, DateTimeField , GeographyField, RecordField.BytesField, RecordField.BooleanField, IntegerArrayField from %s order by TimestampField", + TABLE_ID_FASTQUERY_BQ_RESULTSET.getTable()); ConnectionSettings connectionSettings = ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); Connection connection = bigquery.createConnection(connectionSettings); BigQueryDryRunResult bigQueryDryRunResultSet = connection.dryRun(query); assertNotNull(bigQueryDryRunResultSet.getSchema()); + assertEquals( + BQ_RESULTSET_EXPECTED_SCHEMA, bigQueryDryRunResultSet.getSchema()); // match the schema List queryParameters = bigQueryDryRunResultSet.getQueryParameters(); List expectedQueryParameters = ImmutableList.of( From b82d831196e2de5ee1de0bf74a4df9c311b33af6 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Mon, 17 Jan 2022 17:36:24 +0530 Subject: [PATCH 098/277] Added schema check @ testConnectionImplDryRun --- .../test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index badb500f4..37f7e890f 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -2228,7 +2228,7 @@ public void testQueryExternalHivePartitioningOptionCustomLayout() throws Interru public void testConnectionImplDryRun() throws SQLException { String query = String.format( - "select StringField, BigNumericField, BooleanField, BytesField, IntegerField, TimestampField, FloatField, NumericField, TimeField, DateField, DateTimeField , GeographyField, RecordField.BytesField, RecordField.BooleanField, IntegerArrayField from %s order by TimestampField", + "select StringField, BigNumericField, BooleanField, BytesField, IntegerField, TimestampField, FloatField, NumericField, TimeField, DateField, DateTimeField , GeographyField, RecordField.BytesField, RecordField.BooleanField, IntegerArrayField from %s where StringField = ? order by TimestampField", TABLE_ID_FASTQUERY_BQ_RESULTSET.getTable()); ConnectionSettings connectionSettings = ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); From 2a34bc6ef2b3e9a1b9290f8c9278bc98286b3a3f Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Mon, 17 Jan 2022 20:02:21 +0530 Subject: [PATCH 099/277] Added Java Doc, Removed commented code of processGetQueryResults --- .../google/cloud/bigquery/ConnectionImpl.java | 86 ++++++++----------- 1 file changed, 36 insertions(+), 50 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 644b565eb..68cb488a2 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -75,10 +75,14 @@ final class ConnectionImpl implements Connection { : Math.min(connectionSettings.getNumBufferedRows() * 2, 100000)); } - /* - Cancel method shutdowns the pageFetcher and producerWorker threads gracefully using interrupt. - The pageFetcher threat will not request for any subsequent threads after interrupting and shutdown as soon as any ongoing RPC call returns. - The producerWorker will not populate the buffer with any further records and clear the buffer, put a EoF marker and shutdown. + /** + * Cancel method shutdowns the pageFetcher and producerWorker threads gracefully using interrupt. + * The pageFetcher threat will not request for any subsequent threads after interrupting and + * shutdown as soon as any ongoing RPC call returns. The producerWorker will not populate the + * buffer with any further records and clear the buffer, put a EoF marker and shutdown. + * + * @return Boolean value true if the threads were interrupted + * @throws BigQuerySQLException */ @Override public synchronized Boolean cancel() throws BigQuerySQLException { @@ -86,6 +90,13 @@ public synchronized Boolean cancel() throws BigQuerySQLException { return queryTaskExecutor.isShutdown(); } + /** + * This method runs a dry run query + * + * @param sql SQL SELECT statement + * @return BigQueryDryRunResult containing List and Schema + * @throws BigQuerySQLException + */ @Override public BigQueryDryRunResult dryRun(String sql) throws BigQuerySQLException { com.google.api.services.bigquery.model.Job dryRunJob = createDryRunJob(sql); @@ -95,6 +106,13 @@ public BigQueryDryRunResult dryRun(String sql) throws BigQuerySQLException { return new BigQueryDryRunResultImpl(schema, queryParameters); } + /** + * This method executes a SQL SELECT query + * + * @param sql SQL SELECT statement + * @return BigQueryResultSet containing the output of the query + * @throws BigQuerySQLException + */ @Override public BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException { // use jobs.query if all the properties of connectionSettings are supported @@ -112,6 +130,20 @@ public BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException { firstPage.getTotalRows().longValue(), firstPage.getRows().size(), jobId, firstPage); } + /** + * This method executes a SQL SELECT query + * + * @param sql SQL SELECT query + * @param parameters named or positional parameters. The set of query parameters must either be + * all positional or all named parameters. + * @param labels the labels associated with this query. You can use these to organize and group + * your query jobs. Label keys and values can be no longer than 63 characters, can only + * contain lowercase letters, numeric characters, underscores and dashes. International + * characters are allowed. Label values are optional. Label keys must start with a letter and + * each label in the list must have a different key. + * @return BigQueryResultSet containing the output of the query + * @throws BigQuerySQLException + */ @Override public BigQueryResultSet executeSelect( String sql, List parameters, Map labels) @@ -583,52 +615,6 @@ private GetQueryResultsResponse getQueryResultsFirstPage(JobId jobId) { } } - /* - //TODO(prasmish): We have wired processQueryResponseResults(results, jobId) method instead, delete this after code review - - private BigQueryResultSet processGetQueryResults(JobId jobId, GetQueryResultsResponse results) { - long numRows = results.getTotalRows() == null ? 0 : results.getTotalRows().longValue(); - Schema schema = results.getSchema() == null ? null : Schema.fromPb(results.getSchema()); - - // Get query statistics - Job queryJob = getQueryJobRpc(jobId); - JobStatistics.QueryStatistics statistics = queryJob.getStatistics(); - DmlStats dmlStats = statistics.getDmlStats() == null ? null : statistics.getDmlStats(); - JobStatistics.SessionInfo sessionInfo = - statistics.getSessionInfo() == null ? null : statistics.getSessionInfo(); - BigQueryResultSetStats bigQueryResultSetStats = - new BigQueryResultSetStatsImpl(dmlStats, sessionInfo); - - */ - /* // only use this API for the first page of result - if (results.getPageToken() == null) { - - return processQueryResponseResults(results); - // return new BigQueryResultSetImpl(schema, numRows, null, bigQueryResultSetStats); - return null; // - }*/ - /* - - // results - */ - /* for (TableRow tr : results.getRows()) { - // tr.getF(). - FieldValue.fromPb(tr.getF(), null); - }*/ - /* - // results.getp - - // We need com.google.api.services.bigquery.model.QueryResponse to invoke - // processQueryResponseResults - // processQueryResponseResults(results); - - // use tabledata.list or Read API to fetch subsequent pages of results - long totalRows = results.getTotalRows() == null ? 0 : results.getTotalRows().longValue(); - long pageRows = results.getRows().size(); - - return null; // getQueryResultsWithJobId(totalRows, pageRows, schema, jobId); - }*/ - private boolean isFastQuerySupported() { // TODO: add regex logic to check for scripting return connectionSettings.getClustering() == null From 0ca0724e6416ce5fd2b62694023b6d5db4ce42b4 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 19 Jan 2022 14:54:45 +0530 Subject: [PATCH 100/277] Added SetUp and testExecuteSelect --- .../cloud/bigquery/ConnectionImplTest.java | 99 ++++++++++++++++++- 1 file changed, 98 insertions(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java index 0004c4ae0..790cceb29 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java @@ -16,7 +16,105 @@ package com.google.cloud.bigquery; +import static org.junit.Assert.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +import com.google.api.services.bigquery.model.QueryRequest; +import com.google.api.services.bigquery.model.QueryResponse; +import com.google.api.services.bigquery.model.TableSchema; +import com.google.cloud.ServiceOptions; +import com.google.cloud.bigquery.spi.BigQueryRpcFactory; +import com.google.cloud.bigquery.spi.v2.BigQueryRpc; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) public class ConnectionImplTest { + private BigQueryOptions options; + private BigQueryRpcFactory rpcFactoryMock; + private BigQueryRpc bigqueryRpcMock; + private Connection connectionMock; + private BigQuery bigquery; + private Connection connection; + private static final String PROJECT = "project"; + private static final String DEFAULT_TEST_DATASET = "bigquery_test_dataset"; + private static final String FAST_SQL = + "SELECT county, state_name FROM bigquery_test_dataset.large_data_testing_table limit 2"; + private static final long DEFAULT_PAGE_SIZE = 10000L; + private ConnectionSettings connectionSettings; + private static final Schema FAST_QUERY_SCHEMA = + Schema.of( + Field.newBuilder("country", StandardSQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("state_name", StandardSQLTypeName.BIGNUMERIC) + .setMode(Field.Mode.NULLABLE) + .build()); + private static final TableSchema FAST_QUERY_TABLESCHEMA = FAST_QUERY_SCHEMA.toPb(); + private static final BigQueryResultSet BQ_RS_MOCK_RES = + new BigQueryResultSetImpl(FAST_QUERY_SCHEMA, 2, null, null); + + private BigQueryOptions createBigQueryOptionsForProject( + String project, BigQueryRpcFactory rpcFactory) { + return BigQueryOptions.newBuilder() + .setProjectId(project) + .setServiceRpcFactory(rpcFactory) + .setRetrySettings(ServiceOptions.getNoRetrySettings()) + .build(); + } + + @Before + public void setUp() { + rpcFactoryMock = mock(BigQueryRpcFactory.class); + bigqueryRpcMock = mock(BigQueryRpc.class); + connectionMock = mock(Connection.class); + when(rpcFactoryMock.create(any(BigQueryOptions.class))).thenReturn(bigqueryRpcMock); + options = createBigQueryOptionsForProject(PROJECT, rpcFactoryMock); + bigquery = options.getService(); + + connectionSettings = + ConnectionSettings.newBuilder() + .setDefaultDataset(DatasetId.of(DEFAULT_TEST_DATASET)) + .setNumBufferedRows(DEFAULT_PAGE_SIZE) + .build(); + bigquery = + options + .toBuilder() + .setRetrySettings(ServiceOptions.getDefaultRetrySettings()) + .build() + .getService(); + connection = bigquery.createConnection(connectionSettings); + assertNotNull(connection); + } + + @Test + public void testCancel() throws BigQuerySQLException { + // TODO + } + + @Test + public void testExecuteSelect() throws BigQuerySQLException { + com.google.api.services.bigquery.model.QueryResponse mockQueryRes = + new QueryResponse().setSchema(FAST_QUERY_TABLESCHEMA).setJobComplete(true); + when(bigqueryRpcMock.queryRpc(any(String.class), any(QueryRequest.class))) + .thenReturn(mockQueryRes); + Connection connectionSpy = Mockito.spy(connection); + doReturn(BQ_RS_MOCK_RES) + .when(connectionSpy) + .processQueryResponseResults( + any(com.google.api.services.bigquery.model.QueryResponse.class)); + + BigQueryResultSet res = connectionSpy.executeSelect(FAST_SQL); + assertEquals(res.getTotalRows(), 2); + assertEquals(FAST_QUERY_SCHEMA, res.getSchema()); + verify(connectionSpy, times(1)) + .processQueryResponseResults( + any(com.google.api.services.bigquery.model.QueryResponse.class)); + } // TODO: Add testNextPageTask() @@ -41,5 +139,4 @@ public class ConnectionImplTest { // TODO: Add testQueryDryRun() - // TODO: Add testCancel() } From d911ea7f77f90c04024e0560d50b9865f876c1a7 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 19 Jan 2022 15:22:15 +0530 Subject: [PATCH 101/277] Exposing processQueryResponseResults for testing --- .../src/main/java/com/google/cloud/bigquery/Connection.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java index f254cadc0..a2f71469c 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java @@ -16,6 +16,7 @@ package com.google.cloud.bigquery; +import com.google.api.core.InternalApi; import com.google.api.services.bigquery.model.QueryParameter; import java.util.List; import java.util.Map; @@ -89,4 +90,8 @@ public interface Connection { BigQueryResultSet executeSelect( String sql, List parameters, Map labels) throws BigQuerySQLException; + + @InternalApi("Exposed for testing") + BigQueryResultSet processQueryResponseResults( + com.google.api.services.bigquery.model.QueryResponse results); } From 5e5666a65b43b4c3bbd2e4c4c59a0e52fec981cf Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 19 Jan 2022 15:22:39 +0530 Subject: [PATCH 102/277] Exposing processQueryResponseResults for testing --- .../main/java/com/google/cloud/bigquery/ConnectionImpl.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 68cb488a2..e7a9b0cae 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -19,6 +19,7 @@ import static com.google.cloud.RetryHelper.runWithRetries; import static java.net.HttpURLConnection.HTTP_NOT_FOUND; +import com.google.api.core.InternalApi; import com.google.api.services.bigquery.model.GetQueryResultsResponse; import com.google.api.services.bigquery.model.JobConfigurationQuery; import com.google.api.services.bigquery.model.QueryParameter; @@ -46,7 +47,7 @@ import java.util.stream.Collectors; /** Implementation for {@link Connection}, the generic BigQuery connection API (not JDBC). */ -final class ConnectionImpl implements Connection { +class ConnectionImpl implements Connection { private final ConnectionSettings connectionSettings; private final BigQueryOptions bigQueryOptions; @@ -265,7 +266,8 @@ private BigQueryResultSet tableDataList(GetQueryResultsResponse firstPage, JobId schema, numRows, buffer, bigQueryResultSetStats); } - private BigQueryResultSet processQueryResponseResults( + @InternalApi("Exposed for testing") + public BigQueryResultSet processQueryResponseResults( com.google.api.services.bigquery.model.QueryResponse results) { Schema schema; long numRows; From 49959a5df585c81b1471741d4a610f0be85ac8f1 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 19 Jan 2022 15:41:32 +0530 Subject: [PATCH 103/277] Added testFastQueryMultiplePages and testCancel --- .../cloud/bigquery/ConnectionImplTest.java | 50 ++++++++++++++++--- 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java index 790cceb29..ba8d43fa8 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java @@ -42,6 +42,7 @@ public class ConnectionImplTest { private Connection connection; private static final String PROJECT = "project"; private static final String DEFAULT_TEST_DATASET = "bigquery_test_dataset"; + private static final String PAGE_TOKEN = "ABCD123"; private static final String FAST_SQL = "SELECT county, state_name FROM bigquery_test_dataset.large_data_testing_table limit 2"; private static final long DEFAULT_PAGE_SIZE = 10000L; @@ -58,6 +59,9 @@ public class ConnectionImplTest { private static final BigQueryResultSet BQ_RS_MOCK_RES = new BigQueryResultSetImpl(FAST_QUERY_SCHEMA, 2, null, null); + private static final BigQueryResultSet BQ_RS_MOCK_RES_MULTI_PAGE = + new BigQueryResultSetImpl(FAST_QUERY_SCHEMA, 4, null, null); + private BigQueryOptions createBigQueryOptionsForProject( String project, BigQueryRpcFactory rpcFactory) { return BigQueryOptions.newBuilder() @@ -92,12 +96,7 @@ public void setUp() { } @Test - public void testCancel() throws BigQuerySQLException { - // TODO - } - - @Test - public void testExecuteSelect() throws BigQuerySQLException { + public void testFastQuerySinglePage() throws BigQuerySQLException { com.google.api.services.bigquery.model.QueryResponse mockQueryRes = new QueryResponse().setSchema(FAST_QUERY_TABLESCHEMA).setJobComplete(true); when(bigqueryRpcMock.queryRpc(any(String.class), any(QueryRequest.class))) @@ -116,6 +115,43 @@ public void testExecuteSelect() throws BigQuerySQLException { any(com.google.api.services.bigquery.model.QueryResponse.class)); } + @Test + public void testFastQueryMultiplePages() throws BigQuerySQLException { + com.google.api.services.bigquery.model.QueryResponse mockQueryRes = + new QueryResponse() + .setSchema(FAST_QUERY_TABLESCHEMA) + .setJobComplete(true) + .setPageToken(PAGE_TOKEN); + when(bigqueryRpcMock.queryRpc(any(String.class), any(QueryRequest.class))) + .thenReturn(mockQueryRes); + Connection connectionSpy = Mockito.spy(connection); + + doReturn(BQ_RS_MOCK_RES_MULTI_PAGE) + .when(connectionSpy) + .processQueryResponseResults( + any(com.google.api.services.bigquery.model.QueryResponse.class)); + + BigQueryResultSet res = connectionSpy.executeSelect(FAST_SQL); + assertEquals(res.getTotalRows(), 4); + assertEquals(FAST_QUERY_SCHEMA, res.getSchema()); + verify(connectionSpy, times(1)) + .processQueryResponseResults( + any(com.google.api.services.bigquery.model.QueryResponse.class)); + } + + @Test + public void testCancel() throws BigQuerySQLException { + boolean cancelled = connection.cancel(); + assertTrue(cancelled); + } + + @Test + public void testQueryDryRun() throws BigQuerySQLException { + // TODO + + } + + // Question: Shall I expose these as well for testing and test using some mock/dummy data? // TODO: Add testNextPageTask() // TODO: Add testParseDataTask() @@ -124,8 +160,6 @@ public void testExecuteSelect() throws BigQuerySQLException { // TODO: Add testGetQueryResultsFirstPage() - // TODO: Add testFastQuerySinglePage() - // TODO: Add testFastQueryMultiplePages() --> should exercise processQueryResponseResults() method // TODO: Add testFastQueryLongRunning() --> should exercise getSubsequentQueryResultsWithJob() From 66e947e9746d60329561bd91835efae15275642c Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 19 Jan 2022 17:24:35 +0530 Subject: [PATCH 104/277] Added testQueryDryRun --- .../cloud/bigquery/ConnectionImplTest.java | 38 +++++++++++++++---- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java index ba8d43fa8..1019f284e 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java @@ -19,13 +19,15 @@ import static org.junit.Assert.*; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.*; +import static org.mockito.Mockito.verify; -import com.google.api.services.bigquery.model.QueryRequest; +import com.google.api.services.bigquery.model.*; import com.google.api.services.bigquery.model.QueryResponse; -import com.google.api.services.bigquery.model.TableSchema; import com.google.cloud.ServiceOptions; import com.google.cloud.bigquery.spi.BigQueryRpcFactory; import com.google.cloud.bigquery.spi.v2.BigQueryRpc; +import com.google.common.collect.ImmutableList; +import java.util.List; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -45,6 +47,8 @@ public class ConnectionImplTest { private static final String PAGE_TOKEN = "ABCD123"; private static final String FAST_SQL = "SELECT county, state_name FROM bigquery_test_dataset.large_data_testing_table limit 2"; + private static final String DRY_RUN_SQL = + "SELECT county, state_name FROM bigquery_test_dataset.large_data_testing_table where country = ?"; private static final long DEFAULT_PAGE_SIZE = 10000L; private ConnectionSettings connectionSettings; private static final Schema FAST_QUERY_SCHEMA = @@ -116,6 +120,8 @@ public void testFastQuerySinglePage() throws BigQuerySQLException { } @Test + // NOTE: This doesn't truly paginates. Returns a response while mocking + // processQueryResponseResults public void testFastQueryMultiplePages() throws BigQuerySQLException { com.google.api.services.bigquery.model.QueryResponse mockQueryRes = new QueryResponse() @@ -147,8 +153,28 @@ public void testCancel() throws BigQuerySQLException { @Test public void testQueryDryRun() throws BigQuerySQLException { - // TODO - + List queryParametersMock = + ImmutableList.of( + new QueryParameter().setParameterType(new QueryParameterType().setType("STRING"))); + com.google.api.services.bigquery.model.JobStatistics2 queryMock = + new com.google.api.services.bigquery.model.JobStatistics2() + .setSchema(FAST_QUERY_TABLESCHEMA) + .setUndeclaredQueryParameters(queryParametersMock); + com.google.api.services.bigquery.model.JobStatistics jobStatsMock = + new com.google.api.services.bigquery.model.JobStatistics() + .setCreationTime(1234L) + .setStartTime(5678L) + .setQuery(queryMock); + com.google.api.services.bigquery.model.Job mockDryRunJob = + new com.google.api.services.bigquery.model.Job().setStatistics(jobStatsMock); + + when(bigqueryRpcMock.createJobForQuery(any(com.google.api.services.bigquery.model.Job.class))) + .thenReturn(mockDryRunJob); + BigQueryDryRunResult dryRunResult = connection.dryRun(DRY_RUN_SQL); + assertEquals(1, dryRunResult.getQueryParameters().size()); + assertEquals(FAST_QUERY_SCHEMA, dryRunResult.getSchema()); + verify(bigqueryRpcMock, times(1)) + .createJobForQuery(any(com.google.api.services.bigquery.model.Job.class)); } // Question: Shall I expose these as well for testing and test using some mock/dummy data? @@ -160,8 +186,6 @@ public void testQueryDryRun() throws BigQuerySQLException { // TODO: Add testGetQueryResultsFirstPage() - // TODO: Add testFastQueryMultiplePages() --> should exercise processQueryResponseResults() method - // TODO: Add testFastQueryLongRunning() --> should exercise getSubsequentQueryResultsWithJob() // method @@ -171,6 +195,4 @@ public void testQueryDryRun() throws BigQuerySQLException { // TODO: Add testLegacyQueryMultiplePages() --> should exercise processQueryResponseResults() // method - // TODO: Add testQueryDryRun() - } From 4daeb76d270303877467bf81172f84f93f8731e7 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 20 Jan 2022 17:54:12 +0530 Subject: [PATCH 105/277] Exposing Async methods for testing --- .../com/google/cloud/bigquery/Connection.java | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java index a2f71469c..4f5199d7d 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java @@ -18,8 +18,13 @@ import com.google.api.core.InternalApi; import com.google.api.services.bigquery.model.QueryParameter; +import com.google.api.services.bigquery.model.TableDataList; +import com.google.api.services.bigquery.model.TableRow; +import com.google.cloud.Tuple; +import java.util.AbstractList; import java.util.List; import java.util.Map; +import java.util.concurrent.BlockingQueue; /** * A Connection is a session between a Java application and BigQuery. SQL statements are executed @@ -94,4 +99,23 @@ BigQueryResultSet executeSelect( @InternalApi("Exposed for testing") BigQueryResultSet processQueryResponseResults( com.google.api.services.bigquery.model.QueryResponse results); + + @InternalApi("Exposed for testing") + public void parseRpcDataAsync( + List tableRows, + Schema schema, + BlockingQueue, Boolean>> pageCache, + BlockingQueue> rpcResponseQueue); + + @InternalApi("Exposed for testing") + public void populateBufferAsync( + BlockingQueue> rpcResponseQueue, + BlockingQueue, Boolean>> pageCache, + BlockingQueue> buffer); + + @InternalApi("Exposed for testing") + public void runNextPageTaskAsync( + String firstPageToken, + TableId destinationTable, + BlockingQueue> rpcResponseQueue); } From b32bbdcd02902626763914041451ca7cdd661251 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 20 Jan 2022 17:55:24 +0530 Subject: [PATCH 106/277] Exposing Async methods for testing --- .../java/com/google/cloud/bigquery/ConnectionImpl.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index e7a9b0cae..c6760e729 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -304,7 +304,8 @@ public BigQueryResultSet processQueryResponseResults( schema, numRows, buffer, bigQueryResultSetStats); } - private void runNextPageTaskAsync( + @InternalApi("Exposed for testing") + public void runNextPageTaskAsync( String firstPageToken, TableId destinationTable, BlockingQueue> rpcResponseQueue) { @@ -341,7 +342,8 @@ private void runNextPageTaskAsync( /* This method takes TableDataList from rpcResponseQueue and populates pageCache with FieldValueList */ - private void parseRpcDataAsync( + @InternalApi("Exposed for testing") + public void parseRpcDataAsync( // com.google.api.services.bigquery.model.QueryResponse results, List tableRows, Schema schema, @@ -394,7 +396,8 @@ private void parseRpcDataAsync( queryTaskExecutor.execute(parseDataTask); } - private void populateBufferAsync( + @InternalApi("Exposed for testing") + public void populateBufferAsync( BlockingQueue> rpcResponseQueue, BlockingQueue, Boolean>> pageCache, BlockingQueue> buffer) { From 171da1fc2974ecb47537db92820f246691e63e24 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 20 Jan 2022 18:17:55 +0530 Subject: [PATCH 107/277] Exposing tableDataListRpc for testing --- .../main/java/com/google/cloud/bigquery/ConnectionImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index c6760e729..71cf3e5a9 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -550,7 +550,8 @@ private TableId getDestinationTable(JobId jobId) { return ((QueryJobConfiguration) job.getConfiguration()).getDestinationTable(); } - private TableDataList tableDataListRpc(TableId destinationTable, String pageToken) { + @InternalApi("Exposed for testing") + public TableDataList tableDataListRpc(TableId destinationTable, String pageToken) { try { final TableId completeTableId = destinationTable.setProjectId( From 8971a27ad781259c28379240659a5c464c4116f6 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 20 Jan 2022 18:18:15 +0530 Subject: [PATCH 108/277] Exposed tableDataListRpc for testing --- .../src/main/java/com/google/cloud/bigquery/Connection.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java index 4f5199d7d..dca8b24b1 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java @@ -118,4 +118,7 @@ public void runNextPageTaskAsync( String firstPageToken, TableId destinationTable, BlockingQueue> rpcResponseQueue); + + @InternalApi("Exposed for testing") + public TableDataList tableDataListRpc(TableId destinationTable, String pageToken); } From 9bc67d467d0bf45f410aa49eb976da0d6f14cfdd Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 20 Jan 2022 18:19:00 +0530 Subject: [PATCH 109/277] Added testcases for the 3 async methods --- .../cloud/bigquery/ConnectionImplTest.java | 114 +++++++++++++++++- 1 file changed, 109 insertions(+), 5 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java index 1019f284e..973654647 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java @@ -24,10 +24,14 @@ import com.google.api.services.bigquery.model.*; import com.google.api.services.bigquery.model.QueryResponse; import com.google.cloud.ServiceOptions; +import com.google.cloud.Tuple; import com.google.cloud.bigquery.spi.BigQueryRpcFactory; import com.google.cloud.bigquery.spi.v2.BigQueryRpc; import com.google.common.collect.ImmutableList; +import java.util.AbstractList; import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingDeque; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -45,6 +49,9 @@ public class ConnectionImplTest { private static final String PROJECT = "project"; private static final String DEFAULT_TEST_DATASET = "bigquery_test_dataset"; private static final String PAGE_TOKEN = "ABCD123"; + private static final TableId TABLE_NAME = TableId.of(DEFAULT_TEST_DATASET, PROJECT); + private static final TableCell STRING_CELL = new TableCell().setV("Value"); + private static final TableRow TABLE_ROW = new TableRow().setF(ImmutableList.of(STRING_CELL)); private static final String FAST_SQL = "SELECT county, state_name FROM bigquery_test_dataset.large_data_testing_table limit 2"; private static final String DRY_RUN_SQL = @@ -56,7 +63,7 @@ public class ConnectionImplTest { Field.newBuilder("country", StandardSQLTypeName.STRING) .setMode(Field.Mode.NULLABLE) .build(), - Field.newBuilder("state_name", StandardSQLTypeName.BIGNUMERIC) + Field.newBuilder("state_name", StandardSQLTypeName.STRING) .setMode(Field.Mode.NULLABLE) .build()); private static final TableSchema FAST_QUERY_TABLESCHEMA = FAST_QUERY_SCHEMA.toPb(); @@ -177,12 +184,109 @@ public void testQueryDryRun() throws BigQuerySQLException { .createJobForQuery(any(com.google.api.services.bigquery.model.Job.class)); } - // Question: Shall I expose these as well for testing and test using some mock/dummy data? - // TODO: Add testNextPageTask() + @Test + public void testParseDataTask() throws InterruptedException { + List tableRows = + ImmutableList.of( + new TableRow() + .setF( + ImmutableList.of( + new TableCell().setV("Value1"), new TableCell().setV("Value2"))), + new TableRow() + .setF( + ImmutableList.of( + new TableCell().setV("Value3"), new TableCell().setV("Value4")))); + + BlockingQueue, Boolean>> pageCache = + new LinkedBlockingDeque<>(2); + BlockingQueue> rpcResponseQueue = new LinkedBlockingDeque<>(2); + rpcResponseQueue.offer(Tuple.of(null, false)); + // This call should populate page cache + Connection connectionSpy = Mockito.spy(connection); + connectionSpy.parseRpcDataAsync(tableRows, FAST_QUERY_SCHEMA, pageCache, rpcResponseQueue); + Tuple, Boolean> fvlTupple = + pageCache.take(); // wait for the parser thread to parse the data + assertNotNull(fvlTupple); + Iterable iterableFvl = fvlTupple.x(); + int rowCnt = 0; + for (FieldValueList fvl : iterableFvl) { + assertEquals(2, fvl.size()); // both the rows should have 2 fields each + rowCnt++; + } + assertEquals(2, rowCnt); // row rows read + + verify(connectionSpy, times(1)) + .parseRpcDataAsync( + any(List.class), any(Schema.class), any(BlockingQueue.class), any(BlockingQueue.class)); + } + + @Test + public void testPopulateBuffer() throws InterruptedException { + List tableRows = + ImmutableList.of( + new TableRow() + .setF( + ImmutableList.of( + new TableCell().setV("Value1"), new TableCell().setV("Value2"))), + new TableRow() + .setF( + ImmutableList.of( + new TableCell().setV("Value3"), new TableCell().setV("Value4")))); + + BlockingQueue, Boolean>> pageCache = + new LinkedBlockingDeque<>(2); + BlockingQueue> rpcResponseQueue = new LinkedBlockingDeque<>(2); + BlockingQueue> buffer = new LinkedBlockingDeque<>(5); + rpcResponseQueue.offer(Tuple.of(null, false)); + // This call should populate page cache + Connection connectionSpy = Mockito.spy(connection); + + connectionSpy.parseRpcDataAsync(tableRows, FAST_QUERY_SCHEMA, pageCache, rpcResponseQueue); + + verify(connectionSpy, times(1)) + .parseRpcDataAsync( + any(List.class), any(Schema.class), any(BlockingQueue.class), any(BlockingQueue.class)); - // TODO: Add testParseDataTask() + // now pass the pageCache to populateBuffer method + connectionSpy.populateBufferAsync(rpcResponseQueue, pageCache, buffer); + // check if buffer was populated with two rows async by using the blocking take method + AbstractList fvl1 = buffer.take(); + assertNotNull(fvl1); + assertEquals(2, fvl1.size()); + assertEquals("Value1", fvl1.get(0).getValue().toString()); + assertEquals("Value2", fvl1.get(1).getValue().toString()); + AbstractList fvl2 = buffer.take(); + assertNotNull(fvl2); + assertEquals(2, fvl2.size()); + assertEquals("Value3", fvl2.get(0).getValue().toString()); + assertEquals("Value4", fvl2.get(1).getValue().toString()); + verify(connectionSpy, times(1)) + .populateBufferAsync( + any(BlockingQueue.class), any(BlockingQueue.class), any(BlockingQueue.class)); + } - // TODO: Add testPopulateBuffer() + @Test + public void testNextPageTask() throws InterruptedException { + BlockingQueue> rpcResponseQueue = new LinkedBlockingDeque<>(2); + TableDataList mockTabledataList = + new TableDataList() + .setPageToken(PAGE_TOKEN) + .setRows(ImmutableList.of(TABLE_ROW)) + .setTotalRows(1L); + Connection connectionSpy = Mockito.spy(connection); + doReturn(mockTabledataList) + .when(connectionSpy) + .tableDataListRpc(any(TableId.class), any(String.class)); + connectionSpy.runNextPageTaskAsync(PAGE_TOKEN, TABLE_NAME, rpcResponseQueue); + Tuple tableDataListTuple = rpcResponseQueue.take(); + assertNotNull(tableDataListTuple); + TableDataList tableDataList = tableDataListTuple.x(); + assertNotNull(tableDataList); + assertEquals("ABCD123", tableDataList.getPageToken()); + assertEquals(Long.valueOf(1), tableDataList.getTotalRows()); + verify(connectionSpy, times(1)) + .runNextPageTaskAsync(any(String.class), any(TableId.class), any(BlockingQueue.class)); + } // TODO: Add testGetQueryResultsFirstPage() From d70ad74bd01a175be9b597497b6988979abe7ed3 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 21 Jan 2022 10:48:34 +0530 Subject: [PATCH 110/277] Exposing getQueryResultsFirstPage, isFastQuerySupported and getSubsequentQueryResultsWithJob for testing --- .../java/com/google/cloud/bigquery/Connection.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java index dca8b24b1..14c8bbdde 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java @@ -17,6 +17,7 @@ package com.google.cloud.bigquery; import com.google.api.core.InternalApi; +import com.google.api.services.bigquery.model.GetQueryResultsResponse; import com.google.api.services.bigquery.model.QueryParameter; import com.google.api.services.bigquery.model.TableDataList; import com.google.api.services.bigquery.model.TableRow; @@ -121,4 +122,14 @@ public void runNextPageTaskAsync( @InternalApi("Exposed for testing") public TableDataList tableDataListRpc(TableId destinationTable, String pageToken); + + @InternalApi("Exposed for testing") + public GetQueryResultsResponse getQueryResultsFirstPage(JobId jobId); + + @InternalApi("Exposed for testing") + public boolean isFastQuerySupported(); + + @InternalApi("Exposed for testing") + public BigQueryResultSet getSubsequentQueryResultsWithJob( + Long totalRows, Long pageRows, JobId jobId, GetQueryResultsResponse firstPage); } From 98c65785a51d02ae503f2f866bcca968be7e9e9a Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 21 Jan 2022 10:48:52 +0530 Subject: [PATCH 111/277] Exposing getQueryResultsFirstPage, isFastQuerySupported and getSubsequentQueryResultsWithJob for testing --- .../com/google/cloud/bigquery/ConnectionImpl.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 71cf3e5a9..9104ad520 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -128,7 +128,7 @@ public BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException { JobId jobId = JobId.fromPb(queryJob.getJobReference()); GetQueryResultsResponse firstPage = getQueryResultsFirstPage(jobId); return getSubsequentQueryResultsWithJob( - firstPage.getTotalRows().longValue(), firstPage.getRows().size(), jobId, firstPage); + firstPage.getTotalRows().longValue(), (long) firstPage.getRows().size(), jobId, firstPage); } /** @@ -162,7 +162,7 @@ public BigQueryResultSet executeSelect( JobId jobId = JobId.fromPb(queryJob.getJobReference()); GetQueryResultsResponse firstPage = getQueryResultsFirstPage(jobId); return getSubsequentQueryResultsWithJob( - firstPage.getTotalRows().longValue(), firstPage.getRows().size(), jobId, firstPage); + firstPage.getTotalRows().longValue(), (long) firstPage.getRows().size(), jobId, firstPage); } static class EndOfFieldValueList @@ -504,8 +504,9 @@ private int getPageCacheSize(Long numBufferedRows, long numRows, Schema schema) } /* Returns query results using either tabledata.list or the high throughput Read API */ - private BigQueryResultSet getSubsequentQueryResultsWithJob( - long totalRows, long pageRows, JobId jobId, GetQueryResultsResponse firstPage) { + @InternalApi("Exposed for testing") + public BigQueryResultSet getSubsequentQueryResultsWithJob( + Long totalRows, Long pageRows, JobId jobId, GetQueryResultsResponse firstPage) { TableId destinationTable = getDestinationTable(jobId); return useReadAPI(totalRows, pageRows) ? highThroughPutRead( @@ -584,7 +585,8 @@ private BigQueryResultSet highThroughPutRead(TableId destinationTable) { } /*Returns just the first page of GetQueryResultsResponse using the jobId*/ - private GetQueryResultsResponse getQueryResultsFirstPage(JobId jobId) { + @InternalApi("Exposed for testing") + public GetQueryResultsResponse getQueryResultsFirstPage(JobId jobId) { JobId completeJobId = jobId .setProjectId(bigQueryOptions.getProjectId()) @@ -621,7 +623,8 @@ private GetQueryResultsResponse getQueryResultsFirstPage(JobId jobId) { } } - private boolean isFastQuerySupported() { + @InternalApi("Exposed for testing") + public boolean isFastQuerySupported() { // TODO: add regex logic to check for scripting return connectionSettings.getClustering() == null && connectionSettings.getCreateDisposition() == null From 4251f25f6c9b7fb5c55b14f6d516c574e7d9298e Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 21 Jan 2022 10:49:48 +0530 Subject: [PATCH 112/277] Added testGetQueryResultsFirstPage and testLegacyQuerySinglePage testcases + Minor refactor --- .../cloud/bigquery/ConnectionImplTest.java | 83 +++++++++++++++---- 1 file changed, 67 insertions(+), 16 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java index 973654647..caa28cb1a 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java @@ -28,6 +28,7 @@ import com.google.cloud.bigquery.spi.BigQueryRpcFactory; import com.google.cloud.bigquery.spi.v2.BigQueryRpc; import com.google.common.collect.ImmutableList; +import java.math.BigInteger; import java.util.AbstractList; import java.util.List; import java.util.concurrent.BlockingQueue; @@ -47,18 +48,20 @@ public class ConnectionImplTest { private BigQuery bigquery; private Connection connection; private static final String PROJECT = "project"; + private static final String JOB = "job"; + private static final String LOCATION = "US"; private static final String DEFAULT_TEST_DATASET = "bigquery_test_dataset"; private static final String PAGE_TOKEN = "ABCD123"; private static final TableId TABLE_NAME = TableId.of(DEFAULT_TEST_DATASET, PROJECT); private static final TableCell STRING_CELL = new TableCell().setV("Value"); private static final TableRow TABLE_ROW = new TableRow().setF(ImmutableList.of(STRING_CELL)); - private static final String FAST_SQL = + private static final String SQL_QUERY = "SELECT county, state_name FROM bigquery_test_dataset.large_data_testing_table limit 2"; private static final String DRY_RUN_SQL = "SELECT county, state_name FROM bigquery_test_dataset.large_data_testing_table where country = ?"; private static final long DEFAULT_PAGE_SIZE = 10000L; private ConnectionSettings connectionSettings; - private static final Schema FAST_QUERY_SCHEMA = + private static final Schema QUERY_SCHEMA = Schema.of( Field.newBuilder("country", StandardSQLTypeName.STRING) .setMode(Field.Mode.NULLABLE) @@ -66,12 +69,24 @@ public class ConnectionImplTest { Field.newBuilder("state_name", StandardSQLTypeName.STRING) .setMode(Field.Mode.NULLABLE) .build()); - private static final TableSchema FAST_QUERY_TABLESCHEMA = FAST_QUERY_SCHEMA.toPb(); + private static final TableSchema FAST_QUERY_TABLESCHEMA = QUERY_SCHEMA.toPb(); private static final BigQueryResultSet BQ_RS_MOCK_RES = - new BigQueryResultSetImpl(FAST_QUERY_SCHEMA, 2, null, null); + new BigQueryResultSetImpl(QUERY_SCHEMA, 2, null, null); private static final BigQueryResultSet BQ_RS_MOCK_RES_MULTI_PAGE = - new BigQueryResultSetImpl(FAST_QUERY_SCHEMA, 4, null, null); + new BigQueryResultSetImpl(QUERY_SCHEMA, 4, null, null); + + private static final JobId QUERY_JOB = JobId.of(PROJECT, JOB).setLocation(LOCATION); + private static final GetQueryResultsResponse GET_QUERY_RESULTS_RESPONSE = + new GetQueryResultsResponse() + .setJobReference(QUERY_JOB.toPb()) + .setRows(ImmutableList.of(TABLE_ROW)) + .setJobComplete(true) + .setCacheHit(false) + .setPageToken(PAGE_TOKEN) + .setTotalBytesProcessed(42L) + .setTotalRows(BigInteger.valueOf(1L)) + .setSchema(FAST_QUERY_TABLESCHEMA); private BigQueryOptions createBigQueryOptionsForProject( String project, BigQueryRpcFactory rpcFactory) { @@ -118,9 +133,9 @@ public void testFastQuerySinglePage() throws BigQuerySQLException { .processQueryResponseResults( any(com.google.api.services.bigquery.model.QueryResponse.class)); - BigQueryResultSet res = connectionSpy.executeSelect(FAST_SQL); + BigQueryResultSet res = connectionSpy.executeSelect(SQL_QUERY); assertEquals(res.getTotalRows(), 2); - assertEquals(FAST_QUERY_SCHEMA, res.getSchema()); + assertEquals(QUERY_SCHEMA, res.getSchema()); verify(connectionSpy, times(1)) .processQueryResponseResults( any(com.google.api.services.bigquery.model.QueryResponse.class)); @@ -144,9 +159,9 @@ public void testFastQueryMultiplePages() throws BigQuerySQLException { .processQueryResponseResults( any(com.google.api.services.bigquery.model.QueryResponse.class)); - BigQueryResultSet res = connectionSpy.executeSelect(FAST_SQL); + BigQueryResultSet res = connectionSpy.executeSelect(SQL_QUERY); assertEquals(res.getTotalRows(), 4); - assertEquals(FAST_QUERY_SCHEMA, res.getSchema()); + assertEquals(QUERY_SCHEMA, res.getSchema()); verify(connectionSpy, times(1)) .processQueryResponseResults( any(com.google.api.services.bigquery.model.QueryResponse.class)); @@ -179,7 +194,7 @@ public void testQueryDryRun() throws BigQuerySQLException { .thenReturn(mockDryRunJob); BigQueryDryRunResult dryRunResult = connection.dryRun(DRY_RUN_SQL); assertEquals(1, dryRunResult.getQueryParameters().size()); - assertEquals(FAST_QUERY_SCHEMA, dryRunResult.getSchema()); + assertEquals(QUERY_SCHEMA, dryRunResult.getSchema()); verify(bigqueryRpcMock, times(1)) .createJobForQuery(any(com.google.api.services.bigquery.model.Job.class)); } @@ -203,7 +218,7 @@ public void testParseDataTask() throws InterruptedException { rpcResponseQueue.offer(Tuple.of(null, false)); // This call should populate page cache Connection connectionSpy = Mockito.spy(connection); - connectionSpy.parseRpcDataAsync(tableRows, FAST_QUERY_SCHEMA, pageCache, rpcResponseQueue); + connectionSpy.parseRpcDataAsync(tableRows, QUERY_SCHEMA, pageCache, rpcResponseQueue); Tuple, Boolean> fvlTupple = pageCache.take(); // wait for the parser thread to parse the data assertNotNull(fvlTupple); @@ -241,7 +256,7 @@ public void testPopulateBuffer() throws InterruptedException { // This call should populate page cache Connection connectionSpy = Mockito.spy(connection); - connectionSpy.parseRpcDataAsync(tableRows, FAST_QUERY_SCHEMA, pageCache, rpcResponseQueue); + connectionSpy.parseRpcDataAsync(tableRows, QUERY_SCHEMA, pageCache, rpcResponseQueue); verify(connectionSpy, times(1)) .parseRpcDataAsync( @@ -288,14 +303,50 @@ public void testNextPageTask() throws InterruptedException { .runNextPageTaskAsync(any(String.class), any(TableId.class), any(BlockingQueue.class)); } - // TODO: Add testGetQueryResultsFirstPage() + @Test + public void testGetQueryResultsFirstPage() { + when(bigqueryRpcMock.getQueryResultsWithRowLimit( + any(String.class), any(String.class), any(String.class), any(Long.class))) + .thenReturn(GET_QUERY_RESULTS_RESPONSE); + GetQueryResultsResponse response = connection.getQueryResultsFirstPage(QUERY_JOB); + assertNotNull(response); + assertEquals(GET_QUERY_RESULTS_RESPONSE, response); + verify(bigqueryRpcMock, times(1)) + .getQueryResultsWithRowLimit( + any(String.class), any(String.class), any(String.class), any(Long.class)); + } + + // calls executeSelect with a nonFast query and exercises createQueryJob + @Test + public void testLegacyQuerySinglePage() throws BigQuerySQLException { + Connection connectionSpy = Mockito.spy(connection); + com.google.api.services.bigquery.model.Job jobResponseMock = + new com.google.api.services.bigquery.model.Job() + // .setConfiguration(QUERY_JOB.g) + .setJobReference(QUERY_JOB.toPb()) + .setId(JOB) + .setStatus(new com.google.api.services.bigquery.model.JobStatus().setState("DONE")); + // emulating a legacy query + doReturn(false).when(connectionSpy).isFastQuerySupported(); + doReturn(GET_QUERY_RESULTS_RESPONSE) + .when(connectionSpy) + .getQueryResultsFirstPage(any(JobId.class)); + doReturn(BQ_RS_MOCK_RES) + .when(connectionSpy) + .getSubsequentQueryResultsWithJob( + any(Long.class), any(Long.class), any(JobId.class), any(GetQueryResultsResponse.class)); + when(bigqueryRpcMock.createJobForQuery(any(com.google.api.services.bigquery.model.Job.class))) + .thenReturn(jobResponseMock); // RPC call in createQueryJob + BigQueryResultSet res = connectionSpy.executeSelect(SQL_QUERY); + assertEquals(res.getTotalRows(), 2); + assertEquals(QUERY_SCHEMA, res.getSchema()); + verify(bigqueryRpcMock, times(1)) + .createJobForQuery(any(com.google.api.services.bigquery.model.Job.class)); + } // TODO: Add testFastQueryLongRunning() --> should exercise getSubsequentQueryResultsWithJob() // method - // TODO: Add testLegacyQuerySinglePage() --> should exercise createQueryJob() and - // getQueryResultsFirstPage() - // TODO: Add testLegacyQueryMultiplePages() --> should exercise processQueryResponseResults() // method From cae60ef2c2c8517c47695b14d1f5f3aeb07475c3 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 21 Jan 2022 10:54:24 +0530 Subject: [PATCH 113/277] Deleted TODO --- .../src/main/java/com/google/cloud/bigquery/ConnectionImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 9104ad520..7e1bb99c4 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -219,7 +219,6 @@ private BigQueryResultSet queryRpc(final String projectId, final QueryRequest qu } } - // TODO: Write a IT with order by query /* This method processed the first page of GetQueryResultsResponse and then it uses tabledata.list */ private BigQueryResultSet tableDataList(GetQueryResultsResponse firstPage, JobId jobId) { Schema schema; From 497a8242e19b73338cf7d0a0f31fde86e39a6c4b Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 21 Jan 2022 11:30:55 +0530 Subject: [PATCH 114/277] Exposed getDestinationTable and tableDataList for testing --- .../src/main/java/com/google/cloud/bigquery/Connection.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java index 14c8bbdde..ec0d3639e 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java @@ -132,4 +132,10 @@ public void runNextPageTaskAsync( @InternalApi("Exposed for testing") public BigQueryResultSet getSubsequentQueryResultsWithJob( Long totalRows, Long pageRows, JobId jobId, GetQueryResultsResponse firstPage); + + @InternalApi("Exposed for testing") + public TableId getDestinationTable(JobId jobId); + + @InternalApi("Exposed for testing") + public BigQueryResultSet tableDataList(GetQueryResultsResponse firstPage, JobId jobId); } From cec20bb074fb41028dae4fa642ef0c6e061483b5 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 21 Jan 2022 11:31:13 +0530 Subject: [PATCH 115/277] Exposed getDestinationTable and tableDataList for testing --- .../main/java/com/google/cloud/bigquery/ConnectionImpl.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 7e1bb99c4..49098fbdf 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -220,7 +220,8 @@ private BigQueryResultSet queryRpc(final String projectId, final QueryRequest qu } /* This method processed the first page of GetQueryResultsResponse and then it uses tabledata.list */ - private BigQueryResultSet tableDataList(GetQueryResultsResponse firstPage, JobId jobId) { + @InternalApi("Exposed for testing") + public BigQueryResultSet tableDataList(GetQueryResultsResponse firstPage, JobId jobId) { Schema schema; long numRows; schema = Schema.fromPb(firstPage.getSchema()); @@ -545,7 +546,8 @@ private Job getQueryJobRpc(JobId jobId) { } /* Returns the destinationTable from jobId by calling jobs.get API */ - private TableId getDestinationTable(JobId jobId) { + @InternalApi("Exposed for testing") + public TableId getDestinationTable(JobId jobId) { Job job = getQueryJobRpc(jobId); return ((QueryJobConfiguration) job.getConfiguration()).getDestinationTable(); } From 27a68d08581553a9b1d70e95b28dc4a4eaea003b Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 21 Jan 2022 11:31:36 +0530 Subject: [PATCH 116/277] Added testFastQueryLongRunning testcase --- .../cloud/bigquery/ConnectionImplTest.java | 49 ++++++++++++++++++- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java index caa28cb1a..9118c9636 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java @@ -29,6 +29,7 @@ import com.google.cloud.bigquery.spi.v2.BigQueryRpc; import com.google.common.collect.ImmutableList; import java.math.BigInteger; +import java.sql.SQLException; import java.util.AbstractList; import java.util.List; import java.util.concurrent.BlockingQueue; @@ -344,8 +345,52 @@ public void testLegacyQuerySinglePage() throws BigQuerySQLException { .createJobForQuery(any(com.google.api.services.bigquery.model.Job.class)); } - // TODO: Add testFastQueryLongRunning() --> should exercise getSubsequentQueryResultsWithJob() - // method + // exercises getSubsequentQueryResultsWithJob for fast running queries + @Test + public void testFastQueryLongRunning() throws SQLException { + List tableRows = + ImmutableList.of( + new TableRow() + .setF( + ImmutableList.of( + new TableCell().setV("Value1"), new TableCell().setV("Value2"))), + new TableRow() + .setF( + ImmutableList.of( + new TableCell().setV("Value3"), new TableCell().setV("Value4")))); + Connection connectionSpy = Mockito.spy(connection); + com.google.api.services.bigquery.model.Job jobResponseMock = + new com.google.api.services.bigquery.model.Job() + // .setConfiguration(QUERY_JOB.g) + .setJobReference(QUERY_JOB.toPb()) + .setId(JOB) + .setStatus(new com.google.api.services.bigquery.model.JobStatus().setState("DONE")); + // emulating a fast query + doReturn(true).when(connectionSpy).isFastQuerySupported(); + doReturn(GET_QUERY_RESULTS_RESPONSE) + .when(connectionSpy) + .getQueryResultsFirstPage(any(JobId.class)); + + doReturn(TABLE_NAME).when(connectionSpy).getDestinationTable(any(JobId.class)); + doReturn(BQ_RS_MOCK_RES) + .when(connectionSpy) + .tableDataList(any(GetQueryResultsResponse.class), any(JobId.class)); + + com.google.api.services.bigquery.model.QueryResponse mockQueryRes = + new QueryResponse() + .setSchema(FAST_QUERY_TABLESCHEMA) + .setJobComplete(false) + .setTotalRows(new BigInteger(String.valueOf(4L))) + .setJobReference(QUERY_JOB.toPb()) + .setRows(tableRows); + when(bigqueryRpcMock.queryRpc(any(String.class), any(QueryRequest.class))) + .thenReturn(mockQueryRes); + + BigQueryResultSet res = connectionSpy.executeSelect(SQL_QUERY); + assertEquals(res.getTotalRows(), 2); + assertEquals(QUERY_SCHEMA, res.getSchema()); + verify(bigqueryRpcMock, times(1)).queryRpc(any(String.class), any(QueryRequest.class)); + } // TODO: Add testLegacyQueryMultiplePages() --> should exercise processQueryResponseResults() // method From c3c2d148f1b14dcadb453e784a6d183c074c1568 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 21 Jan 2022 14:10:05 +0530 Subject: [PATCH 117/277] Added testLegacyQueryMultiplePages --- .../cloud/bigquery/ConnectionImplTest.java | 42 ++++++++++++++----- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java index 9118c9636..e1264e7e8 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java @@ -359,12 +359,6 @@ public void testFastQueryLongRunning() throws SQLException { ImmutableList.of( new TableCell().setV("Value3"), new TableCell().setV("Value4")))); Connection connectionSpy = Mockito.spy(connection); - com.google.api.services.bigquery.model.Job jobResponseMock = - new com.google.api.services.bigquery.model.Job() - // .setConfiguration(QUERY_JOB.g) - .setJobReference(QUERY_JOB.toPb()) - .setId(JOB) - .setStatus(new com.google.api.services.bigquery.model.JobStatus().setState("DONE")); // emulating a fast query doReturn(true).when(connectionSpy).isFastQuerySupported(); doReturn(GET_QUERY_RESULTS_RESPONSE) @@ -385,14 +379,42 @@ public void testFastQueryLongRunning() throws SQLException { .setRows(tableRows); when(bigqueryRpcMock.queryRpc(any(String.class), any(QueryRequest.class))) .thenReturn(mockQueryRes); - BigQueryResultSet res = connectionSpy.executeSelect(SQL_QUERY); assertEquals(res.getTotalRows(), 2); assertEquals(QUERY_SCHEMA, res.getSchema()); verify(bigqueryRpcMock, times(1)).queryRpc(any(String.class), any(QueryRequest.class)); } - // TODO: Add testLegacyQueryMultiplePages() --> should exercise processQueryResponseResults() - // method - + @Test + // Emulates first page response using getQueryResultsFirstPage(jobId) and then subsequent pages + // using getQueryResultsFirstPage(jobId) getSubsequentQueryResultsWithJob( + public void testLegacyQueryMultiplePages() throws SQLException { + Connection connectionSpy = Mockito.spy(connection); + com.google.api.services.bigquery.model.JobStatistics jobStatistics = + new com.google.api.services.bigquery.model.JobStatistics(); + // emulating a Legacy query + doReturn(false).when(connectionSpy).isFastQuerySupported(); + doReturn(GET_QUERY_RESULTS_RESPONSE) + .when(connectionSpy) + .getQueryResultsFirstPage(any(JobId.class)); + doReturn(TABLE_NAME).when(connectionSpy).getDestinationTable(any(JobId.class)); + doReturn(BQ_RS_MOCK_RES) + .when(connectionSpy) + .tableDataList(any(GetQueryResultsResponse.class), any(JobId.class)); + com.google.api.services.bigquery.model.Job jobResponseMock = + new com.google.api.services.bigquery.model.Job() + .setJobReference(QUERY_JOB.toPb()) + .setId(JOB) + .setStatus(new com.google.api.services.bigquery.model.JobStatus().setState("DONE")) + .setStatistics(jobStatistics); + when(bigqueryRpcMock.createJobForQuery(any(com.google.api.services.bigquery.model.Job.class))) + .thenReturn(jobResponseMock); // RPC call in createQueryJob + BigQueryResultSet res = connectionSpy.executeSelect(SQL_QUERY); + assertEquals(res.getTotalRows(), 2); + assertEquals(QUERY_SCHEMA, res.getSchema()); + verify(bigqueryRpcMock, times(1)) + .createJobForQuery(any(com.google.api.services.bigquery.model.Job.class)); + verify(connectionSpy, times(1)) + .tableDataList(any(GetQueryResultsResponse.class), any(JobId.class)); + } } From 5bbb06613b37e752cdafe48f35addfd0dd6ca870 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 2 Feb 2022 21:05:22 +0530 Subject: [PATCH 118/277] Added BigqueryStorage and Arrow's dependencies --- pom.xml | 62 +++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 45 insertions(+), 17 deletions(-) diff --git a/pom.xml b/pom.xml index 9c6aba59e..8d9b35ed1 100644 --- a/pom.xml +++ b/pom.xml @@ -66,25 +66,32 @@ pom import + + + + com.google.cloud + google-cloud-datacatalog-bom + 1.6.1 + pom + import + - - com.google.cloud - google-cloud-datacatalog-bom - 1.6.1 - pom - import - - - - org.checkerframework - checker-compat-qual - 2.5.5 - + + org.checkerframework + checker-compat-qual + 2.5.5 + - - com.google.cloud - google-cloud-bigquery - 2.5.2-SNAPSHOT + + com.google.cloud + google-cloud-bigquery + 2.5.2-SNAPSHOT @@ -134,6 +141,27 @@ + + + + com.google.cloud + google-cloud-bigquerystorage + 2.8.3 + + + org.apache.arrow + arrow-vector + 5.0.0 + + + + org.apache.arrow + arrow-memory-netty + 5.0.0 + runtime + + + google-cloud-bigquery From 92e8883aac7356417853cbd1bb9dbe360307ba19 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 2 Feb 2022 21:05:52 +0530 Subject: [PATCH 119/277] Added draft impl for highThroughPutRead --- .../google/cloud/bigquery/ConnectionImpl.java | 162 +++++++++++++++++- 1 file changed, 157 insertions(+), 5 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 49098fbdf..7891c1ff1 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -29,15 +29,15 @@ import com.google.cloud.RetryHelper; import com.google.cloud.Tuple; import com.google.cloud.bigquery.spi.v2.BigQueryRpc; +import com.google.cloud.bigquery.storage.v1.*; import com.google.common.base.Function; +import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; -import java.util.AbstractList; -import java.util.List; -import java.util.Map; -import java.util.UUID; +import java.io.IOException; +import java.util.*; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -45,6 +45,13 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.*; +import org.apache.arrow.vector.ipc.ReadChannel; +import org.apache.arrow.vector.ipc.message.MessageSerializer; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.util.ByteArrayReadableSeekableByteChannel; /** Implementation for {@link Connection}, the generic BigQuery connection API (not JDBC). */ class ConnectionImpl implements Connection { @@ -581,10 +588,155 @@ public TableDataList tableDataListRpc(TableId destinationTable, String pageToken } } + private BigQueryReadClient + bqReadClient; // TODO - Check if this instance be reused, currently it's being initialized on + // every call + // query result should be saved in the destination table private BigQueryResultSet highThroughPutRead(TableId destinationTable) { - return null; + + try { + bqReadClient = BigQueryReadClient.create(); + String parent = String.format("projects/%s", destinationTable.getProject()); + String srcTable = + String.format( + "projects/%s/datasets/%s/tables/%s", + destinationTable.getProject(), + destinationTable.getDataset(), + destinationTable.getProject()); + + /* ReadSession.TableReadOptions options = + ReadSession.TableReadOptions.newBuilder(). + .build();*/ + + // Read all the columns if the source table and stream the data back in Arrow format + ReadSession.Builder sessionBuilder = + ReadSession.newBuilder().setTable(srcTable).setDataFormat(DataFormat.ARROW) + // .setReadOptions(options)//TODO: Check if entire table be read if we are not specifying + // the options + ; + + CreateReadSessionRequest.Builder builder = + CreateReadSessionRequest.newBuilder() + .setParent(parent) + .setReadSession(sessionBuilder) + .setMaxStreamCount(1) // Currently just one stream is allowed + // DO a regex check using order by and use multiple streams + ; + + ReadSession readSession = bqReadClient.createReadSession(builder.build()); + + // TODO(prasmish) Modify the type in the Tuple, dynamically determine the capacity + BlockingQueue> buffer = new LinkedBlockingDeque<>(500); + + // deserialize and populate the buffer async, so that the client isn't blocked + processArrowStreamAsync(readSession, buffer); + + // TODO(prasmish) pass the right value + return new BigQueryResultSetImpl>(null, -1, buffer, null); + + } catch (IOException e) { + // Throw Error + e.printStackTrace(); + throw new RuntimeException(e.getMessage()); // TODO exception handling + } + + // return null; + } + + private void processArrowStreamAsync( + ReadSession readSession, BlockingQueue> buffer) { + + Runnable arrowStreamProcessor = + () -> { + try (SimpleRowReader reader = new SimpleRowReader(readSession.getArrowSchema())) { + // Use the first stream to perform reading. + String streamName = readSession.getStreams(0).getName(); + + ReadRowsRequest readRowsRequest = + ReadRowsRequest.newBuilder().setReadStream(streamName).build(); + + // Process each block of rows as they arrive and decode using our simple row reader. + com.google.api.gax.rpc.ServerStream stream = + bqReadClient.readRowsCallable().call(readRowsRequest); + for (ReadRowsResponse response : stream) { + reader.processRows(response.getArrowRecordBatch(), buffer); + } + } catch (Exception e) { + + } + }; + + queryTaskExecutor.execute(arrowStreamProcessor); } + // TODO(prasmish) - refactor + private ArrowSchema arrowSchema; + + /* + * SimpleRowReader handles deserialization of the Apache Arrow-encoded row batches transmitted + * from the storage API using a generic datum decoder. + */ + private static class SimpleRowReader implements AutoCloseable { + + BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); + + // Decoder object will be reused to avoid re-allocation and too much garbage collection. + private final VectorSchemaRoot root; + private final VectorLoader loader; + + public SimpleRowReader(ArrowSchema arrowSchema) throws IOException { + org.apache.arrow.vector.types.pojo.Schema schema = + MessageSerializer.deserializeSchema( + new org.apache.arrow.vector.ipc.ReadChannel( + new ByteArrayReadableSeekableByteChannel( + arrowSchema.getSerializedSchema().toByteArray()))); + Preconditions.checkNotNull(schema); + List vectors = new ArrayList<>(); + for (Field field : schema.getFields()) { + vectors.add(field.createVector(allocator)); + } + root = new VectorSchemaRoot(vectors); + loader = new VectorLoader(root); + } + + /** @param batch object returned from the ReadRowsResponse. */ + public void processRows(ArrowRecordBatch batch, BlockingQueue> buffer) + throws IOException { // deserialize the values and consume the hash of the values + org.apache.arrow.vector.ipc.message.ArrowRecordBatch deserializedBatch = + MessageSerializer.deserializeRecordBatch( + new ReadChannel( + new ByteArrayReadableSeekableByteChannel( + batch.getSerializedRecordBatch().toByteArray())), + allocator); + + loader.load(deserializedBatch); + // Release buffers from batch (they are still held in the vectors in root). + deserializedBatch.close(); + + for (int i = 0; i < root.getRowCount(); i++) { + + try { + buffer.put( + Tuple.of(new String(((VarCharVector) root.getVector("vendor_id")).get(i)), true)); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + root.clear(); + try { + buffer.put(Tuple.of(null, false)); // marking End of the stream + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + @Override + public void close() { + root.close(); + allocator.close(); + } + } /*Returns just the first page of GetQueryResultsResponse using the jobId*/ @InternalApi("Exposed for testing") public GetQueryResultsResponse getQueryResultsFirstPage(JobId jobId) { From b0bf13a97c5237526ccf457f43fa5c7647804070 Mon Sep 17 00:00:00 2001 From: stephwang Date: Wed, 2 Feb 2022 17:44:05 -0500 Subject: [PATCH 120/277] resolve dependency issues --- pom.xml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index cd4a269de..d4b38d8aa 100644 --- a/pom.xml +++ b/pom.xml @@ -157,22 +157,29 @@ + + com.google.protobuf + protobuf-java + com.google.cloud google-cloud-bigquerystorage 2.8.3 + + com.google.api.grpc + proto-google-cloud-bigquerystorage-v1 + 2.8.3 + org.apache.arrow arrow-vector - 5.0.0 + 6.0.0 - org.apache.arrow - arrow-memory-netty - 5.0.0 - runtime + arrow-memory-core + 6.0.0 From c1743c90e9ba51360fcececcdaac4cd42507bb7d Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Wed, 2 Feb 2022 23:09:01 +0000 Subject: [PATCH 121/277] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0b3f49656..31e1b32d3 100644 --- a/README.md +++ b/README.md @@ -59,13 +59,13 @@ implementation 'com.google.cloud:google-cloud-bigquery' If you are using Gradle without BOM, add this to your dependencies ```Groovy -implementation 'com.google.cloud:google-cloud-bigquery:2.7.1' +implementation 'com.google.cloud:google-cloud-bigquery:2.8.0' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.7.1" +libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.8.0" ``` ## Authentication From 3297e4d09f12e2f12cd48e2e2927e880342dc1ea Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 8 Feb 2022 12:15:40 +0530 Subject: [PATCH 122/277] Added Attribute to for ArrowSchema --- .../cloud/bigquery/BigQueryResultSetImpl.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index 3d0df7e4b..6ae795e14 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -16,6 +16,7 @@ package com.google.cloud.bigquery; +import com.google.cloud.bigquery.storage.v1.ArrowSchema; import java.math.BigDecimal; import java.sql.Date; import java.sql.ResultSet; @@ -28,7 +29,8 @@ // TODO: This implementation deals with the JSON response. We can have respective implementations public class BigQueryResultSetImpl implements BigQueryResultSet { - private final Schema schema; + private Schema schema; + private ArrowSchema arrowSchema; private final long totalRows; private final BlockingQueue buffer; private T cursor; @@ -36,11 +38,16 @@ public class BigQueryResultSetImpl implements BigQueryResultSet { private final BigQueryResultSetStats bigQueryResultSetStats; public BigQueryResultSetImpl( - Schema schema, + Object schema, long totalRows, BlockingQueue buffer, BigQueryResultSetStats bigQueryResultSetStats) { - this.schema = schema; + if (schema instanceof Schema) { + this.schema = (Schema) schema; + } else if (schema instanceof ArrowSchema) { + this.arrowSchema = (ArrowSchema) schema; + } + this.totalRows = totalRows; this.buffer = buffer; this.underlyingResultSet = new ResultSetWrapper(); @@ -52,6 +59,8 @@ public Schema getSchema() { return schema; } + // TODO(prasmish): Implement a method to return ArrowSchema + @Override public long getTotalRows() { return totalRows; From ca244ebdc30ffe15771cc393eb55b6c2bcb66f7e Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 8 Feb 2022 12:16:18 +0530 Subject: [PATCH 123/277] updated highThroughPutRead --- .../google/cloud/bigquery/ConnectionImpl.java | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 7891c1ff1..dac4be0bb 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -226,6 +226,15 @@ private BigQueryResultSet queryRpc(final String projectId, final QueryRequest qu } } + private BigQueryResultSetStats getBigQueryResultSetStats(JobId jobId) { + // Create GetQueryResultsResponse query statistics + Job queryJob = getQueryJobRpc(jobId); + JobStatistics.QueryStatistics statistics = queryJob.getStatistics(); + DmlStats dmlStats = statistics.getDmlStats() == null ? null : statistics.getDmlStats(); + JobStatistics.SessionInfo sessionInfo = + statistics.getSessionInfo() == null ? null : statistics.getSessionInfo(); + return new BigQueryResultSetStatsImpl(dmlStats, sessionInfo); + } /* This method processed the first page of GetQueryResultsResponse and then it uses tabledata.list */ @InternalApi("Exposed for testing") public BigQueryResultSet tableDataList(GetQueryResultsResponse firstPage, JobId jobId) { @@ -234,14 +243,7 @@ public BigQueryResultSet tableDataList(GetQueryResultsResponse firstPage, JobId schema = Schema.fromPb(firstPage.getSchema()); numRows = firstPage.getTotalRows().longValue(); - // Create GetQueryResultsResponse query statistics - Job queryJob = getQueryJobRpc(jobId); - JobStatistics.QueryStatistics statistics = queryJob.getStatistics(); - DmlStats dmlStats = statistics.getDmlStats() == null ? null : statistics.getDmlStats(); - JobStatistics.SessionInfo sessionInfo = - statistics.getSessionInfo() == null ? null : statistics.getSessionInfo(); - BigQueryResultSetStats bigQueryResultSetStats = - new BigQueryResultSetStatsImpl(dmlStats, sessionInfo); + BigQueryResultSetStats bigQueryResultSetStats = getBigQueryResultSetStats(jobId); // Keeps the deserialized records at the row level, which is consumed by BigQueryResultSet BlockingQueue> buffer = new LinkedBlockingDeque<>(bufferSize); @@ -517,7 +519,10 @@ public BigQueryResultSet getSubsequentQueryResultsWithJob( TableId destinationTable = getDestinationTable(jobId); return useReadAPI(totalRows, pageRows) ? highThroughPutRead( - destinationTable) // discord first page and stream the entire BigQueryResultSet using + destinationTable, + firstPage.getTotalRows().longValue(), + getBigQueryResultSetStats( + jobId)) // discord first page and stream the entire BigQueryResultSet using // the Read API : tableDataList(firstPage, jobId); } @@ -592,7 +597,8 @@ public TableDataList tableDataListRpc(TableId destinationTable, String pageToken bqReadClient; // TODO - Check if this instance be reused, currently it's being initialized on // every call // query result should be saved in the destination table - private BigQueryResultSet highThroughPutRead(TableId destinationTable) { + private BigQueryResultSet highThroughPutRead( + TableId destinationTable, long totalRows, BigQueryResultSetStats stats) { try { bqReadClient = BigQueryReadClient.create(); @@ -631,16 +637,14 @@ private BigQueryResultSet highThroughPutRead(TableId destinationTable) { // deserialize and populate the buffer async, so that the client isn't blocked processArrowStreamAsync(readSession, buffer); - // TODO(prasmish) pass the right value - return new BigQueryResultSetImpl>(null, -1, buffer, null); + return new BigQueryResultSetImpl>( + readSession.getArrowSchema(), totalRows, buffer, stats); } catch (IOException e) { // Throw Error e.printStackTrace(); throw new RuntimeException(e.getMessage()); // TODO exception handling } - - // return null; } private void processArrowStreamAsync( From 7cded43359a2ab7543c52479ed7a0acd014b272f Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 8 Feb 2022 18:40:03 +0530 Subject: [PATCH 124/277] Revert "Added Attribute to for ArrowSchema" This reverts commit 3297e4d09f12e2f12cd48e2e2927e880342dc1ea. --- .../cloud/bigquery/BigQueryResultSetImpl.java | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index 6ae795e14..3d0df7e4b 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -16,7 +16,6 @@ package com.google.cloud.bigquery; -import com.google.cloud.bigquery.storage.v1.ArrowSchema; import java.math.BigDecimal; import java.sql.Date; import java.sql.ResultSet; @@ -29,8 +28,7 @@ // TODO: This implementation deals with the JSON response. We can have respective implementations public class BigQueryResultSetImpl implements BigQueryResultSet { - private Schema schema; - private ArrowSchema arrowSchema; + private final Schema schema; private final long totalRows; private final BlockingQueue buffer; private T cursor; @@ -38,16 +36,11 @@ public class BigQueryResultSetImpl implements BigQueryResultSet { private final BigQueryResultSetStats bigQueryResultSetStats; public BigQueryResultSetImpl( - Object schema, + Schema schema, long totalRows, BlockingQueue buffer, BigQueryResultSetStats bigQueryResultSetStats) { - if (schema instanceof Schema) { - this.schema = (Schema) schema; - } else if (schema instanceof ArrowSchema) { - this.arrowSchema = (ArrowSchema) schema; - } - + this.schema = schema; this.totalRows = totalRows; this.buffer = buffer; this.underlyingResultSet = new ResultSetWrapper(); @@ -59,8 +52,6 @@ public Schema getSchema() { return schema; } - // TODO(prasmish): Implement a method to return ArrowSchema - @Override public long getTotalRows() { return totalRows; From 33920a772e3bb92cdec27ba47959e6111a119a3e Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 8 Feb 2022 20:28:04 +0530 Subject: [PATCH 125/277] composed BigQueryResultSetImpl with all the params --- .../com/google/cloud/bigquery/ConnectionImpl.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index dac4be0bb..51e0d6885 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -26,6 +26,7 @@ import com.google.api.services.bigquery.model.QueryRequest; import com.google.api.services.bigquery.model.TableDataList; import com.google.api.services.bigquery.model.TableRow; +import com.google.api.services.bigquery.model.TableSchema; import com.google.cloud.RetryHelper; import com.google.cloud.Tuple; import com.google.cloud.bigquery.spi.v2.BigQueryRpc; @@ -521,6 +522,7 @@ public BigQueryResultSet getSubsequentQueryResultsWithJob( ? highThroughPutRead( destinationTable, firstPage.getTotalRows().longValue(), + firstPage.getSchema(), getBigQueryResultSetStats( jobId)) // discord first page and stream the entire BigQueryResultSet using // the Read API @@ -598,7 +600,10 @@ public TableDataList tableDataListRpc(TableId destinationTable, String pageToken // every call // query result should be saved in the destination table private BigQueryResultSet highThroughPutRead( - TableId destinationTable, long totalRows, BigQueryResultSetStats stats) { + TableId destinationTable, + long totalRows, + TableSchema tableSchema, + BigQueryResultSetStats stats) { try { bqReadClient = BigQueryReadClient.create(); @@ -637,8 +642,8 @@ private BigQueryResultSet highThroughPutRead( // deserialize and populate the buffer async, so that the client isn't blocked processArrowStreamAsync(readSession, buffer); - return new BigQueryResultSetImpl>( - readSession.getArrowSchema(), totalRows, buffer, stats); + Schema schema = Schema.fromPb(tableSchema); + return new BigQueryResultSetImpl>(schema, totalRows, buffer, stats); } catch (IOException e) { // Throw Error @@ -718,6 +723,9 @@ public void processRows(ArrowRecordBatch batch, BlockingQueue Date: Thu, 10 Feb 2022 19:36:54 +0530 Subject: [PATCH 126/277] Added parsing logic for ArrowVectors --- .../google/cloud/bigquery/ConnectionImpl.java | 93 +++++++++++++------ 1 file changed, 65 insertions(+), 28 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 51e0d6885..2943e14c3 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -283,7 +283,6 @@ public BigQueryResultSet processQueryResponseResults( long numRows; schema = Schema.fromPb(results.getSchema()); numRows = results.getTotalRows().longValue(); - // Create QueryResponse query statistics DmlStats dmlStats = results.getDmlStats() == null ? null : DmlStats.fromPb(results.getDmlStats()); @@ -637,13 +636,19 @@ private BigQueryResultSet highThroughPutRead( ReadSession readSession = bqReadClient.createReadSession(builder.build()); // TODO(prasmish) Modify the type in the Tuple, dynamically determine the capacity - BlockingQueue> buffer = new LinkedBlockingDeque<>(500); + BlockingQueue, Boolean>> buffer = new LinkedBlockingDeque<>(500); + Map arrowNameToIndex = new HashMap<>(); + Schema schema = Schema.fromPb(tableSchema); // deserialize and populate the buffer async, so that the client isn't blocked - processArrowStreamAsync(readSession, buffer); + processArrowStreamAsync( + readSession, + buffer, + new ArrowRowReader(readSession.getArrowSchema(), arrowNameToIndex), + schema); - Schema schema = Schema.fromPb(tableSchema); - return new BigQueryResultSetImpl>(schema, totalRows, buffer, stats); + return new BigQueryResultSetImpl, Boolean>>( + schema, totalRows, buffer, stats); } catch (IOException e) { // Throw Error @@ -653,11 +658,15 @@ private BigQueryResultSet highThroughPutRead( } private void processArrowStreamAsync( - ReadSession readSession, BlockingQueue> buffer) { + ReadSession readSession, + BlockingQueue, Boolean>> buffer, + ArrowRowReader reader, + Schema schema) { Runnable arrowStreamProcessor = () -> { - try (SimpleRowReader reader = new SimpleRowReader(readSession.getArrowSchema())) { + try // (ArrowRowReader reader = new ArrowRowReader(readSession.getArrowSchema())) + { // Use the first stream to perform reading. String streamName = readSession.getStreams(0).getName(); @@ -668,8 +677,9 @@ private void processArrowStreamAsync( com.google.api.gax.rpc.ServerStream stream = bqReadClient.readRowsCallable().call(readRowsRequest); for (ReadRowsResponse response : stream) { - reader.processRows(response.getArrowRecordBatch(), buffer); + reader.processRows(response.getArrowRecordBatch(), buffer, schema); } + buffer.put(Tuple.of(null, false)); // marking end of stream } catch (Exception e) { } @@ -678,14 +688,7 @@ private void processArrowStreamAsync( queryTaskExecutor.execute(arrowStreamProcessor); } - // TODO(prasmish) - refactor - private ArrowSchema arrowSchema; - - /* - * SimpleRowReader handles deserialization of the Apache Arrow-encoded row batches transmitted - * from the storage API using a generic datum decoder. - */ - private static class SimpleRowReader implements AutoCloseable { + private static class ArrowRowReader implements AutoCloseable { BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); @@ -693,7 +696,8 @@ private static class SimpleRowReader implements AutoCloseable { private final VectorSchemaRoot root; private final VectorLoader loader; - public SimpleRowReader(ArrowSchema arrowSchema) throws IOException { + public ArrowRowReader(ArrowSchema arrowSchema, Map arrowNameToIndex) + throws IOException { org.apache.arrow.vector.types.pojo.Schema schema = MessageSerializer.deserializeSchema( new org.apache.arrow.vector.ipc.ReadChannel( @@ -701,15 +705,22 @@ public SimpleRowReader(ArrowSchema arrowSchema) throws IOException { arrowSchema.getSerializedSchema().toByteArray()))); Preconditions.checkNotNull(schema); List vectors = new ArrayList<>(); - for (Field field : schema.getFields()) { - vectors.add(field.createVector(allocator)); + List fields = schema.getFields(); + for (int i = 0; i < fields.size(); i++) { + vectors.add(fields.get(i).createVector(allocator)); + arrowNameToIndex.put( + fields.get(i).getName(), + i); // mapping for getting against the field name in the result set } root = new VectorSchemaRoot(vectors); loader = new VectorLoader(root); } /** @param batch object returned from the ReadRowsResponse. */ - public void processRows(ArrowRecordBatch batch, BlockingQueue> buffer) + public void processRows( + ArrowRecordBatch batch, + BlockingQueue, Boolean>> buffer, + Schema schema) throws IOException { // deserialize the values and consume the hash of the values org.apache.arrow.vector.ipc.message.ArrowRecordBatch deserializedBatch = MessageSerializer.deserializeRecordBatch( @@ -722,16 +733,42 @@ public void processRows(ArrowRecordBatch batch, BlockingQueue curRow = new HashMap<>(); + for (int col = 0; col < fields.size(); col++) { // iterate all the vectors for a given row + com.google.cloud.bigquery.Field field = fields.get(col); + // System.out.printf("field.getName() %s, field.getType() %s", field.getName(), + // field.getType() == LegacySQLTypeName.DATE); + FieldVector curFieldVec = + root.getVector( + field.getName()); // can be accessed using the index or Vector/column name + // Now cast the FieldVector depending on the type + if (field.getType() == LegacySQLTypeName.STRING) { + VarCharVector varCharVector = (VarCharVector) curFieldVec; + curRow.put( + field.getName(), + new String(varCharVector.get(rowNum))); // store the row:value mapping + } else if (field.getType() == LegacySQLTypeName.TIMESTAMP) { + TimeStampMicroVector timeStampMicroVector = (TimeStampMicroVector) curFieldVec; + curRow.put( + field.getName(), timeStampMicroVector.get(rowNum)); // store the row:value mapping + } else if (field.getType() == LegacySQLTypeName.INTEGER) { + BigIntVector bigIntVector = (BigIntVector) curFieldVec; + curRow.put(field.getName(), bigIntVector.get(rowNum)); // store the row:value mapping + } else { + throw new RuntimeException("TODO: Implement remaining support type conversions"); + } + } try { - buffer.put( - Tuple.of(new String(((VarCharVector) root.getVector("vendor_id")).get(i)), true)); + buffer.put(Tuple.of(curRow, true)); } catch (InterruptedException e) { - e.printStackTrace(); + e.printStackTrace(); // TODO: Exception handling } } From f411699381d41b4d83ac3d1d70750c25671b74e2 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 11 Feb 2022 17:42:02 +0530 Subject: [PATCH 127/277] Update processRows --- .../main/java/com/google/cloud/bigquery/ConnectionImpl.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 2943e14c3..2e8e41927 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -743,8 +743,6 @@ public void processRows( Map curRow = new HashMap<>(); for (int col = 0; col < fields.size(); col++) { // iterate all the vectors for a given row com.google.cloud.bigquery.Field field = fields.get(col); - // System.out.printf("field.getName() %s, field.getType() %s", field.getName(), - // field.getType() == LegacySQLTypeName.DATE); FieldVector curFieldVec = root.getVector( field.getName()); // can be accessed using the index or Vector/column name @@ -760,7 +758,8 @@ public void processRows( field.getName(), timeStampMicroVector.get(rowNum)); // store the row:value mapping } else if (field.getType() == LegacySQLTypeName.INTEGER) { BigIntVector bigIntVector = (BigIntVector) curFieldVec; - curRow.put(field.getName(), bigIntVector.get(rowNum)); // store the row:value mapping + curRow.put( + field.getName(), (int) bigIntVector.get(rowNum)); // store the row:value mapping } else { throw new RuntimeException("TODO: Implement remaining support type conversions"); } From 39ed02025d034d17b9f5376fea2159771404aae7 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 11 Feb 2022 17:43:07 +0530 Subject: [PATCH 128/277] updated next methods and getters for 3 data types --- .../cloud/bigquery/BigQueryResultSetImpl.java | 36 ++++++++++++++----- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index 3d0df7e4b..35a337eb4 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -16,6 +16,7 @@ package com.google.cloud.bigquery; +import com.google.cloud.Tuple; import java.math.BigDecimal; import java.sql.Date; import java.sql.ResultSet; @@ -23,6 +24,7 @@ import java.sql.Time; import java.sql.Timestamp; import java.time.LocalTime; +import java.util.Map; import java.util.concurrent.BlockingQueue; // TODO: This implementation deals with the JSON response. We can have respective implementations @@ -34,6 +36,7 @@ public class BigQueryResultSetImpl implements BigQueryResultSet { private T cursor; private final ResultSetWrapper underlyingResultSet; private final BigQueryResultSetStats bigQueryResultSetStats; + private final FieldList schemaFieldList; public BigQueryResultSetImpl( Schema schema, @@ -45,6 +48,7 @@ public BigQueryResultSetImpl( this.buffer = buffer; this.underlyingResultSet = new ResultSetWrapper(); this.bigQueryResultSetStats = bigQueryResultSetStats; + this.schemaFieldList = schema.getFields(); } @Override @@ -71,6 +75,12 @@ public boolean next() throws SQLException { if (isEndOfStream(cursor)) { // check for end of stream cursor = null; return false; + } else if (cursor instanceof Tuple) { + Tuple, Boolean> curTup = (Tuple, Boolean>) cursor; + if (!curTup.y()) { // last Tuple + cursor = null; + return false; + } } } catch (InterruptedException e) { throw new SQLException("Error occurred while reading buffer"); @@ -80,9 +90,7 @@ public boolean next() throws SQLException { } private boolean isEndOfStream(T cursor) { - return cursor - instanceof - ConnectionImpl.EndOfFieldValueList; // TODO: Similar check will be required for Arrow + return cursor instanceof ConnectionImpl.EndOfFieldValueList; } @Override @@ -120,8 +128,10 @@ public String getString(String fieldName) throws SQLException { } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); return fieldValue.getValue() == null ? null : fieldValue.getStringValue(); + } else { // Data received from Read API (Arrow) + Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; + return (String) curTuple.x().get(fieldName); } - return null; // TODO: Implementation for Arrow } @Override @@ -131,8 +141,9 @@ public String getString(int columnIndex) throws SQLException { } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); return fieldValue.getValue() == null ? null : fieldValue.getStringValue(); + } else { // Data received from Read API (Arrow) + return getString(schemaFieldList.get(columnIndex).getName()); } - return null; // TODO: Implementation for Arrow } @Override @@ -146,8 +157,10 @@ public int getInt(String fieldName) throws SQLException { } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); return fieldValue.getValue() == null ? 0 : fieldValue.getNumericValue().intValue(); + } else { + Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; + return (Integer) curTuple.x().get(fieldName); } - return 0; // TODO: Implementation for Arrow } @Override @@ -158,8 +171,9 @@ public int getInt(int columnIndex) throws SQLException { } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); return fieldValue.getValue() == null ? 0 : fieldValue.getNumericValue().intValue(); + } else { // Data received from Read API (Arrow) + return getInt(schemaFieldList.get(columnIndex).getName()); } - return 0; // TODO: Implementation for Arrow } @Override @@ -314,8 +328,11 @@ public Timestamp getTimestamp(String fieldName) throws SQLException { fieldValue.getTimestampValue() / 1000); // getTimestampValue returns time in microseconds, and TimeStamp // expects it in millis + } else { + Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; + return new Timestamp( + (Long) curTuple.x().get(fieldName) / 1000); // Timestamp is represented as a Long } - return null; // TODO: Implementation for Arrow } @Override @@ -330,8 +347,9 @@ public Timestamp getTimestamp(int columnIndex) throws SQLException { fieldValue.getTimestampValue() / 1000); // getTimestampValue returns time in microseconds, and TimeStamp // expects it in millis + } else { + return getTimestamp(schemaFieldList.get(columnIndex).getName()); } - return null; // TODO: Implementation for Arrow } @Override From 99301efe1d1485a147eb6cad5c31680ad76a4a72 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 15 Feb 2022 12:21:19 +0530 Subject: [PATCH 129/277] Added translateAndThrow(Exception ex) --- .../java/com/google/cloud/bigquery/BigQueryException.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryException.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryException.java index 06cbf344c..818c0beb1 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryException.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryException.java @@ -134,4 +134,8 @@ static BaseServiceException translateAndThrow(ExecutionException ex) { BaseServiceException.translate(ex); throw new BigQueryException(UNKNOWN_CODE, ex.getMessage(), ex.getCause()); } + + static BaseServiceException translateAndThrow(Exception ex) { + throw new BigQueryException(UNKNOWN_CODE, ex.getMessage(), ex.getCause()); + } } From 19318bb8d8bcfd4a6644abf30123c91af2cbfc2d Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 15 Feb 2022 12:21:52 +0530 Subject: [PATCH 130/277] Refactored processRows, added exception handling --- .../google/cloud/bigquery/ConnectionImpl.java | 108 +++++++++--------- 1 file changed, 55 insertions(+), 53 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 2e8e41927..bf8c49d3a 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -32,7 +32,6 @@ import com.google.cloud.bigquery.spi.v2.BigQueryRpc; import com.google.cloud.bigquery.storage.v1.*; import com.google.common.base.Function; -import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; @@ -681,14 +680,15 @@ private void processArrowStreamAsync( } buffer.put(Tuple.of(null, false)); // marking end of stream } catch (Exception e) { - + throw BigQueryException.translateAndThrow(e); } }; queryTaskExecutor.execute(arrowStreamProcessor); } - private static class ArrowRowReader implements AutoCloseable { + private class ArrowRowReader + implements AutoCloseable { // TODO: Update to recent version of Arrow to avoid memoryleak BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); @@ -703,7 +703,6 @@ public ArrowRowReader(ArrowSchema arrowSchema, Map arrowNameToI new org.apache.arrow.vector.ipc.ReadChannel( new ByteArrayReadableSeekableByteChannel( arrowSchema.getSerializedSchema().toByteArray()))); - Preconditions.checkNotNull(schema); List vectors = new ArrayList<>(); List fields = schema.getFields(); for (int i = 0; i < fields.size(); i++) { @@ -722,60 +721,63 @@ public void processRows( BlockingQueue, Boolean>> buffer, Schema schema) throws IOException { // deserialize the values and consume the hash of the values - org.apache.arrow.vector.ipc.message.ArrowRecordBatch deserializedBatch = - MessageSerializer.deserializeRecordBatch( - new ReadChannel( - new ByteArrayReadableSeekableByteChannel( - batch.getSerializedRecordBatch().toByteArray())), - allocator); - - loader.load(deserializedBatch); - // Release buffers from batch (they are still held in the vectors in root). - deserializedBatch.close(); - - // Parse the vectors using BQ Schema. Deserialize the data at the row level and add it to the - // buffer - FieldList fields = schema.getFields(); - for (int rowNum = 0; - rowNum < root.getRowCount(); - rowNum++) { // for the given number of rows in the batch - - Map curRow = new HashMap<>(); - for (int col = 0; col < fields.size(); col++) { // iterate all the vectors for a given row - com.google.cloud.bigquery.Field field = fields.get(col); - FieldVector curFieldVec = - root.getVector( - field.getName()); // can be accessed using the index or Vector/column name - // Now cast the FieldVector depending on the type - if (field.getType() == LegacySQLTypeName.STRING) { - VarCharVector varCharVector = (VarCharVector) curFieldVec; - curRow.put( - field.getName(), - new String(varCharVector.get(rowNum))); // store the row:value mapping - } else if (field.getType() == LegacySQLTypeName.TIMESTAMP) { - TimeStampMicroVector timeStampMicroVector = (TimeStampMicroVector) curFieldVec; - curRow.put( - field.getName(), timeStampMicroVector.get(rowNum)); // store the row:value mapping - } else if (field.getType() == LegacySQLTypeName.INTEGER) { - BigIntVector bigIntVector = (BigIntVector) curFieldVec; - curRow.put( - field.getName(), (int) bigIntVector.get(rowNum)); // store the row:value mapping - } else { - throw new RuntimeException("TODO: Implement remaining support type conversions"); + try { + org.apache.arrow.vector.ipc.message.ArrowRecordBatch deserializedBatch = + MessageSerializer.deserializeRecordBatch( + new ReadChannel( + new ByteArrayReadableSeekableByteChannel( + batch.getSerializedRecordBatch().toByteArray())), + allocator); + + loader.load(deserializedBatch); + // Release buffers from batch (they are still held in the vectors in root). + deserializedBatch.close(); + + // Parse the vectors using BQ Schema. Deserialize the data at the row level and add it to + // the + // buffer + FieldList fields = schema.getFields(); + for (int rowNum = 0; + rowNum < root.getRowCount(); + rowNum++) { // for the given number of rows in the batch + + Map curRow = new HashMap<>(); + for (int col = 0; col < fields.size(); col++) { // iterate all the vectors for a given row + com.google.cloud.bigquery.Field field = fields.get(col); + FieldVector curFieldVec = + root.getVector( + field.getName()); // can be accessed using the index or Vector/column name + // Now cast the FieldVector depending on the type + if (field.getType() == LegacySQLTypeName.STRING) { + VarCharVector varCharVector = (VarCharVector) curFieldVec; + curRow.put( + field.getName(), + new String(varCharVector.get(rowNum))); // store the row:value mapping + } else if (field.getType() == LegacySQLTypeName.TIMESTAMP) { + TimeStampMicroVector timeStampMicroVector = (TimeStampMicroVector) curFieldVec; + curRow.put( + field.getName(), timeStampMicroVector.get(rowNum)); // store the row:value mapping + } else if (field.getType() == LegacySQLTypeName.INTEGER) { + BigIntVector bigIntVector = (BigIntVector) curFieldVec; + curRow.put( + field.getName(), (int) bigIntVector.get(rowNum)); // store the row:value mapping + } else { + throw new RuntimeException("TODO: Implement remaining support type conversions"); + } } - } - try { buffer.put(Tuple.of(curRow, true)); - } catch (InterruptedException e) { - e.printStackTrace(); // TODO: Exception handling } - } + root.clear(); // TODO: make sure to clear the root while implementing the thread + // interruption logic (Connection.close method) - root.clear(); - try { - buffer.put(Tuple.of(null, false)); // marking End of the stream } catch (InterruptedException e) { - e.printStackTrace(); + throw BigQueryException.translateAndThrow(e); + } finally { + try { + root.clear(); + } catch (RuntimeException e) { + logger.log(Level.WARNING, "\n Error while clearing VectorSchemaRoot ", e); + } } } From 0fb29069d648602db080c87849d573927f653881 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 15 Feb 2022 16:40:58 +0530 Subject: [PATCH 131/277] Added arrow-memory-netty --- pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pom.xml b/pom.xml index d4b38d8aa..22e447754 100644 --- a/pom.xml +++ b/pom.xml @@ -181,6 +181,12 @@ arrow-memory-core 6.0.0 + + org.apache.arrow + arrow-memory-netty + 7.0.0 + runtime + From 8714d892a38800ca120268a0ff6150c0ce9aab74 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 15 Feb 2022 16:49:50 +0530 Subject: [PATCH 132/277] Updated highThroughPutRead - Code fix and refactoring --- .../google/cloud/bigquery/ConnectionImpl.java | 37 ++++++------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index bf8c49d3a..3679287c4 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -65,6 +65,7 @@ class ConnectionImpl implements Connection { private final ExecutorService queryTaskExecutor = Executors.newFixedThreadPool(MAX_PROCESS_QUERY_THREADS_CNT); private final Logger logger = Logger.getLogger(this.getClass().getName()); + private BigQueryReadClient bqReadClient; ConnectionImpl( ConnectionSettings connectionSettings, @@ -593,10 +594,6 @@ public TableDataList tableDataListRpc(TableId destinationTable, String pageToken } } - private BigQueryReadClient - bqReadClient; // TODO - Check if this instance be reused, currently it's being initialized on - // every call - // query result should be saved in the destination table private BigQueryResultSet highThroughPutRead( TableId destinationTable, long totalRows, @@ -604,25 +601,21 @@ private BigQueryResultSet highThroughPutRead( BigQueryResultSetStats stats) { try { - bqReadClient = BigQueryReadClient.create(); + if (bqReadClient == null) { // if the read client isn't already initialized. Not thread safe. + bqReadClient = BigQueryReadClient.create(); + } String parent = String.format("projects/%s", destinationTable.getProject()); String srcTable = String.format( "projects/%s/datasets/%s/tables/%s", destinationTable.getProject(), destinationTable.getDataset(), - destinationTable.getProject()); - - /* ReadSession.TableReadOptions options = - ReadSession.TableReadOptions.newBuilder(). - .build();*/ + destinationTable.getTable()); - // Read all the columns if the source table and stream the data back in Arrow format + // Read all the columns if the source table (temp table) and stream the data back in Arrow + // format ReadSession.Builder sessionBuilder = - ReadSession.newBuilder().setTable(srcTable).setDataFormat(DataFormat.ARROW) - // .setReadOptions(options)//TODO: Check if entire table be read if we are not specifying - // the options - ; + ReadSession.newBuilder().setTable(srcTable).setDataFormat(DataFormat.ARROW); CreateReadSessionRequest.Builder builder = CreateReadSessionRequest.newBuilder() @@ -633,10 +626,8 @@ private BigQueryResultSet highThroughPutRead( ; ReadSession readSession = bqReadClient.createReadSession(builder.build()); - - // TODO(prasmish) Modify the type in the Tuple, dynamically determine the capacity - BlockingQueue, Boolean>> buffer = new LinkedBlockingDeque<>(500); - + BlockingQueue, Boolean>> buffer = + new LinkedBlockingDeque<>(bufferSize); Map arrowNameToIndex = new HashMap<>(); Schema schema = Schema.fromPb(tableSchema); // deserialize and populate the buffer async, so that the client isn't blocked @@ -650,9 +641,7 @@ private BigQueryResultSet highThroughPutRead( schema, totalRows, buffer, stats); } catch (IOException e) { - // Throw Error - e.printStackTrace(); - throw new RuntimeException(e.getMessage()); // TODO exception handling + throw BigQueryException.translateAndThrow(e); } } @@ -664,11 +653,9 @@ private void processArrowStreamAsync( Runnable arrowStreamProcessor = () -> { - try // (ArrowRowReader reader = new ArrowRowReader(readSession.getArrowSchema())) - { + try { // Use the first stream to perform reading. String streamName = readSession.getStreams(0).getName(); - ReadRowsRequest readRowsRequest = ReadRowsRequest.newBuilder().setReadStream(streamName).build(); From d1e3b64be5d9507ee2e9c570d0921cc8dfffbccb Mon Sep 17 00:00:00 2001 From: stephwang Date: Wed, 16 Feb 2022 16:50:32 -0500 Subject: [PATCH 133/277] clean up Connection.java and remove methods meant only for testing from the public interface. Update ConnectionImplTest.java to test against ConnectionImpl instead of Connection. --- .../com/google/cloud/bigquery/Connection.java | 49 -------------- .../google/cloud/bigquery/ConnectionImpl.java | 65 ++++++++++++------- .../cloud/bigquery/ConnectionImplTest.java | 23 ++++--- 3 files changed, 52 insertions(+), 85 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java index ec0d3639e..f254cadc0 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java @@ -16,16 +16,9 @@ package com.google.cloud.bigquery; -import com.google.api.core.InternalApi; -import com.google.api.services.bigquery.model.GetQueryResultsResponse; import com.google.api.services.bigquery.model.QueryParameter; -import com.google.api.services.bigquery.model.TableDataList; -import com.google.api.services.bigquery.model.TableRow; -import com.google.cloud.Tuple; -import java.util.AbstractList; import java.util.List; import java.util.Map; -import java.util.concurrent.BlockingQueue; /** * A Connection is a session between a Java application and BigQuery. SQL statements are executed @@ -96,46 +89,4 @@ public interface Connection { BigQueryResultSet executeSelect( String sql, List parameters, Map labels) throws BigQuerySQLException; - - @InternalApi("Exposed for testing") - BigQueryResultSet processQueryResponseResults( - com.google.api.services.bigquery.model.QueryResponse results); - - @InternalApi("Exposed for testing") - public void parseRpcDataAsync( - List tableRows, - Schema schema, - BlockingQueue, Boolean>> pageCache, - BlockingQueue> rpcResponseQueue); - - @InternalApi("Exposed for testing") - public void populateBufferAsync( - BlockingQueue> rpcResponseQueue, - BlockingQueue, Boolean>> pageCache, - BlockingQueue> buffer); - - @InternalApi("Exposed for testing") - public void runNextPageTaskAsync( - String firstPageToken, - TableId destinationTable, - BlockingQueue> rpcResponseQueue); - - @InternalApi("Exposed for testing") - public TableDataList tableDataListRpc(TableId destinationTable, String pageToken); - - @InternalApi("Exposed for testing") - public GetQueryResultsResponse getQueryResultsFirstPage(JobId jobId); - - @InternalApi("Exposed for testing") - public boolean isFastQuerySupported(); - - @InternalApi("Exposed for testing") - public BigQueryResultSet getSubsequentQueryResultsWithJob( - Long totalRows, Long pageRows, JobId jobId, GetQueryResultsResponse firstPage); - - @InternalApi("Exposed for testing") - public TableId getDestinationTable(JobId jobId); - - @InternalApi("Exposed for testing") - public BigQueryResultSet tableDataList(GetQueryResultsResponse firstPage, JobId jobId); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 3679287c4..29f634640 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -19,7 +19,6 @@ import static com.google.cloud.RetryHelper.runWithRetries; import static java.net.HttpURLConnection.HTTP_NOT_FOUND; -import com.google.api.core.InternalApi; import com.google.api.services.bigquery.model.GetQueryResultsResponse; import com.google.api.services.bigquery.model.JobConfigurationQuery; import com.google.api.services.bigquery.model.QueryParameter; @@ -30,14 +29,27 @@ import com.google.cloud.RetryHelper; import com.google.cloud.Tuple; import com.google.cloud.bigquery.spi.v2.BigQueryRpc; -import com.google.cloud.bigquery.storage.v1.*; +import com.google.cloud.bigquery.storage.v1.ArrowRecordBatch; +import com.google.cloud.bigquery.storage.v1.ArrowSchema; +import com.google.cloud.bigquery.storage.v1.BigQueryReadClient; +import com.google.cloud.bigquery.storage.v1.CreateReadSessionRequest; +import com.google.cloud.bigquery.storage.v1.DataFormat; +import com.google.cloud.bigquery.storage.v1.ReadRowsRequest; +import com.google.cloud.bigquery.storage.v1.ReadRowsResponse; +import com.google.cloud.bigquery.storage.v1.ReadSession; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Function; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import java.io.IOException; -import java.util.*; +import java.util.AbstractList; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -47,7 +59,12 @@ import java.util.stream.Collectors; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; -import org.apache.arrow.vector.*; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.TimeStampMicroVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorLoader; +import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.ipc.ReadChannel; import org.apache.arrow.vector.ipc.message.MessageSerializer; import org.apache.arrow.vector.types.pojo.Field; @@ -237,8 +254,8 @@ private BigQueryResultSetStats getBigQueryResultSetStats(JobId jobId) { return new BigQueryResultSetStatsImpl(dmlStats, sessionInfo); } /* This method processed the first page of GetQueryResultsResponse and then it uses tabledata.list */ - @InternalApi("Exposed for testing") - public BigQueryResultSet tableDataList(GetQueryResultsResponse firstPage, JobId jobId) { + @VisibleForTesting + BigQueryResultSet tableDataList(GetQueryResultsResponse firstPage, JobId jobId) { Schema schema; long numRows; schema = Schema.fromPb(firstPage.getSchema()); @@ -276,8 +293,8 @@ public BigQueryResultSet tableDataList(GetQueryResultsResponse firstPage, JobId schema, numRows, buffer, bigQueryResultSetStats); } - @InternalApi("Exposed for testing") - public BigQueryResultSet processQueryResponseResults( + @VisibleForTesting + BigQueryResultSet processQueryResponseResults( com.google.api.services.bigquery.model.QueryResponse results) { Schema schema; long numRows; @@ -313,8 +330,8 @@ public BigQueryResultSet processQueryResponseResults( schema, numRows, buffer, bigQueryResultSetStats); } - @InternalApi("Exposed for testing") - public void runNextPageTaskAsync( + @VisibleForTesting + void runNextPageTaskAsync( String firstPageToken, TableId destinationTable, BlockingQueue> rpcResponseQueue) { @@ -351,8 +368,8 @@ public void runNextPageTaskAsync( /* This method takes TableDataList from rpcResponseQueue and populates pageCache with FieldValueList */ - @InternalApi("Exposed for testing") - public void parseRpcDataAsync( + @VisibleForTesting + void parseRpcDataAsync( // com.google.api.services.bigquery.model.QueryResponse results, List tableRows, Schema schema, @@ -405,8 +422,8 @@ public void parseRpcDataAsync( queryTaskExecutor.execute(parseDataTask); } - @InternalApi("Exposed for testing") - public void populateBufferAsync( + @VisibleForTesting + void populateBufferAsync( BlockingQueue> rpcResponseQueue, BlockingQueue, Boolean>> pageCache, BlockingQueue> buffer) { @@ -513,8 +530,8 @@ private int getPageCacheSize(Long numBufferedRows, long numRows, Schema schema) } /* Returns query results using either tabledata.list or the high throughput Read API */ - @InternalApi("Exposed for testing") - public BigQueryResultSet getSubsequentQueryResultsWithJob( + @VisibleForTesting + BigQueryResultSet getSubsequentQueryResultsWithJob( Long totalRows, Long pageRows, JobId jobId, GetQueryResultsResponse firstPage) { TableId destinationTable = getDestinationTable(jobId); return useReadAPI(totalRows, pageRows) @@ -559,14 +576,14 @@ private Job getQueryJobRpc(JobId jobId) { } /* Returns the destinationTable from jobId by calling jobs.get API */ - @InternalApi("Exposed for testing") - public TableId getDestinationTable(JobId jobId) { + @VisibleForTesting + TableId getDestinationTable(JobId jobId) { Job job = getQueryJobRpc(jobId); return ((QueryJobConfiguration) job.getConfiguration()).getDestinationTable(); } - @InternalApi("Exposed for testing") - public TableDataList tableDataListRpc(TableId destinationTable, String pageToken) { + @VisibleForTesting + TableDataList tableDataListRpc(TableId destinationTable, String pageToken) { try { final TableId completeTableId = destinationTable.setProjectId( @@ -775,8 +792,8 @@ public void close() { } } /*Returns just the first page of GetQueryResultsResponse using the jobId*/ - @InternalApi("Exposed for testing") - public GetQueryResultsResponse getQueryResultsFirstPage(JobId jobId) { + @VisibleForTesting + GetQueryResultsResponse getQueryResultsFirstPage(JobId jobId) { JobId completeJobId = jobId .setProjectId(bigQueryOptions.getProjectId()) @@ -813,8 +830,8 @@ public GetQueryResultsResponse getQueryResultsFirstPage(JobId jobId) { } } - @InternalApi("Exposed for testing") - public boolean isFastQuerySupported() { + @VisibleForTesting + boolean isFastQuerySupported() { // TODO: add regex logic to check for scripting return connectionSettings.getClustering() == null && connectionSettings.getCreateDisposition() == null diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java index e1264e7e8..3beff4ebc 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java @@ -47,7 +47,7 @@ public class ConnectionImplTest { private BigQueryRpc bigqueryRpcMock; private Connection connectionMock; private BigQuery bigquery; - private Connection connection; + private ConnectionImpl connection; private static final String PROJECT = "project"; private static final String JOB = "job"; private static final String LOCATION = "US"; @@ -118,7 +118,7 @@ public void setUp() { .setRetrySettings(ServiceOptions.getDefaultRetrySettings()) .build() .getService(); - connection = bigquery.createConnection(connectionSettings); + connection = (ConnectionImpl) bigquery.createConnection(connectionSettings); assertNotNull(connection); } @@ -128,11 +128,10 @@ public void testFastQuerySinglePage() throws BigQuerySQLException { new QueryResponse().setSchema(FAST_QUERY_TABLESCHEMA).setJobComplete(true); when(bigqueryRpcMock.queryRpc(any(String.class), any(QueryRequest.class))) .thenReturn(mockQueryRes); - Connection connectionSpy = Mockito.spy(connection); + ConnectionImpl connectionSpy = Mockito.spy(connection); doReturn(BQ_RS_MOCK_RES) .when(connectionSpy) - .processQueryResponseResults( - any(com.google.api.services.bigquery.model.QueryResponse.class)); + .processQueryResponseResults(any(QueryResponse.class)); BigQueryResultSet res = connectionSpy.executeSelect(SQL_QUERY); assertEquals(res.getTotalRows(), 2); @@ -153,7 +152,7 @@ public void testFastQueryMultiplePages() throws BigQuerySQLException { .setPageToken(PAGE_TOKEN); when(bigqueryRpcMock.queryRpc(any(String.class), any(QueryRequest.class))) .thenReturn(mockQueryRes); - Connection connectionSpy = Mockito.spy(connection); + ConnectionImpl connectionSpy = Mockito.spy(connection); doReturn(BQ_RS_MOCK_RES_MULTI_PAGE) .when(connectionSpy) @@ -218,7 +217,7 @@ public void testParseDataTask() throws InterruptedException { BlockingQueue> rpcResponseQueue = new LinkedBlockingDeque<>(2); rpcResponseQueue.offer(Tuple.of(null, false)); // This call should populate page cache - Connection connectionSpy = Mockito.spy(connection); + ConnectionImpl connectionSpy = Mockito.spy(connection); connectionSpy.parseRpcDataAsync(tableRows, QUERY_SCHEMA, pageCache, rpcResponseQueue); Tuple, Boolean> fvlTupple = pageCache.take(); // wait for the parser thread to parse the data @@ -255,7 +254,7 @@ public void testPopulateBuffer() throws InterruptedException { BlockingQueue> buffer = new LinkedBlockingDeque<>(5); rpcResponseQueue.offer(Tuple.of(null, false)); // This call should populate page cache - Connection connectionSpy = Mockito.spy(connection); + ConnectionImpl connectionSpy = Mockito.spy(connection); connectionSpy.parseRpcDataAsync(tableRows, QUERY_SCHEMA, pageCache, rpcResponseQueue); @@ -289,7 +288,7 @@ public void testNextPageTask() throws InterruptedException { .setPageToken(PAGE_TOKEN) .setRows(ImmutableList.of(TABLE_ROW)) .setTotalRows(1L); - Connection connectionSpy = Mockito.spy(connection); + ConnectionImpl connectionSpy = Mockito.spy(connection); doReturn(mockTabledataList) .when(connectionSpy) .tableDataListRpc(any(TableId.class), any(String.class)); @@ -320,7 +319,7 @@ public void testGetQueryResultsFirstPage() { // calls executeSelect with a nonFast query and exercises createQueryJob @Test public void testLegacyQuerySinglePage() throws BigQuerySQLException { - Connection connectionSpy = Mockito.spy(connection); + ConnectionImpl connectionSpy = Mockito.spy(connection); com.google.api.services.bigquery.model.Job jobResponseMock = new com.google.api.services.bigquery.model.Job() // .setConfiguration(QUERY_JOB.g) @@ -358,7 +357,7 @@ public void testFastQueryLongRunning() throws SQLException { .setF( ImmutableList.of( new TableCell().setV("Value3"), new TableCell().setV("Value4")))); - Connection connectionSpy = Mockito.spy(connection); + ConnectionImpl connectionSpy = Mockito.spy(connection); // emulating a fast query doReturn(true).when(connectionSpy).isFastQuerySupported(); doReturn(GET_QUERY_RESULTS_RESPONSE) @@ -389,7 +388,7 @@ public void testFastQueryLongRunning() throws SQLException { // Emulates first page response using getQueryResultsFirstPage(jobId) and then subsequent pages // using getQueryResultsFirstPage(jobId) getSubsequentQueryResultsWithJob( public void testLegacyQueryMultiplePages() throws SQLException { - Connection connectionSpy = Mockito.spy(connection); + ConnectionImpl connectionSpy = Mockito.spy(connection); com.google.api.services.bigquery.model.JobStatistics jobStatistics = new com.google.api.services.bigquery.model.JobStatistics(); // emulating a Legacy query From 30cbd57434e8cbc50c7718c35d272bbf06923400 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 22 Feb 2022 10:45:40 +0530 Subject: [PATCH 134/277] Updated to StandardSQLTypeName --- .../main/java/com/google/cloud/bigquery/ConnectionImpl.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 29f634640..595ccee34 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -752,16 +752,16 @@ public void processRows( root.getVector( field.getName()); // can be accessed using the index or Vector/column name // Now cast the FieldVector depending on the type - if (field.getType() == LegacySQLTypeName.STRING) { + if (field.getType().getStandardType() == StandardSQLTypeName.STRING) { VarCharVector varCharVector = (VarCharVector) curFieldVec; curRow.put( field.getName(), new String(varCharVector.get(rowNum))); // store the row:value mapping - } else if (field.getType() == LegacySQLTypeName.TIMESTAMP) { + } else if (field.getType().getStandardType() == StandardSQLTypeName.TIMESTAMP) { TimeStampMicroVector timeStampMicroVector = (TimeStampMicroVector) curFieldVec; curRow.put( field.getName(), timeStampMicroVector.get(rowNum)); // store the row:value mapping - } else if (field.getType() == LegacySQLTypeName.INTEGER) { + } else if (field.getType().getStandardType() == StandardSQLTypeName.INT64) { BigIntVector bigIntVector = (BigIntVector) curFieldVec; curRow.put( field.getName(), (int) bigIntVector.get(rowNum)); // store the row:value mapping From f0dd594cae8f82abe6d0a45d52d8414098b4c8a5 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 23 Feb 2022 21:46:28 +0530 Subject: [PATCH 135/277] Added Arrow specific logic for primitive getters --- .../cloud/bigquery/BigQueryResultSetImpl.java | 39 +++++++++++++------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index 35a337eb4..fba0229e6 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -157,9 +157,10 @@ public int getInt(String fieldName) throws SQLException { } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); return fieldValue.getValue() == null ? 0 : fieldValue.getNumericValue().intValue(); - } else { + } else { // Data received from Read API (Arrow) Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; - return (Integer) curTuple.x().get(fieldName); + return ((Long) curTuple.x().get(fieldName)) + .intValue(); // BigIntVector.get(...) returns Long } } @@ -187,8 +188,10 @@ public long getLong(String fieldName) throws SQLException { } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); return fieldValue.getValue() == null ? 0L : fieldValue.getNumericValue().longValue(); + } else { // Data received from Read API (Arrow) + Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; + return (Long) curTuple.x().get(fieldName); // BigIntVector.get(...) returns Long } - return 0L; // TODO: Implementation for Arrow } @Override @@ -199,8 +202,9 @@ public long getLong(int columnIndex) throws SQLException { } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); return fieldValue.getValue() == null ? 0L : fieldValue.getNumericValue().longValue(); + } else { // Data received from Read API (Arrow) + return getInt(schemaFieldList.get(columnIndex).getName()); } - return 0L; // TODO: Implementation for Arrow } @Override @@ -214,8 +218,10 @@ public double getDouble(String fieldName) throws SQLException { } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); return fieldValue.getValue() == null ? 0d : fieldValue.getNumericValue().doubleValue(); + } else { // Data received from Read API (Arrow) + Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; + return (Double) curTuple.x().get(fieldName); } - return 0d; // TODO: Implementation for Arrow } @Override @@ -226,8 +232,9 @@ public double getDouble(int columnIndex) throws SQLException { } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); return fieldValue.getValue() == null ? 0d : fieldValue.getNumericValue().doubleValue(); + } else { // Data received from Read API (Arrow) + return getDouble(schemaFieldList.get(columnIndex).getName()); } - return 0d; // TODO: Implementation for Arrow } @Override @@ -244,8 +251,9 @@ public BigDecimal getBigDecimal(String fieldName) throws SQLException { return fieldValue.getValue() == null ? null : BigDecimal.valueOf(fieldValue.getNumericValue().doubleValue()); + } else { // Data received from Read API (Arrow) + return BigDecimal.valueOf(getDouble(fieldName)); } - return null; // TODO: Implementation for Arrow } @Override @@ -259,8 +267,9 @@ public BigDecimal getBigDecimal(int columnIndex) throws SQLException { return fieldValue.getValue() == null ? null : BigDecimal.valueOf(fieldValue.getNumericValue().doubleValue()); + } else { // Data received from Read API (Arrow) + return getBigDecimal(schemaFieldList.get(columnIndex).getName()); } - return null; // TODO: Implementation for Arrow } @Override @@ -273,8 +282,10 @@ public boolean getBoolean(String fieldName) throws SQLException { } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); return fieldValue.getValue() != null && fieldValue.getBooleanValue(); + } else { // Data received from Read API (Arrow) + Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; + return (Boolean) curTuple.x().get(fieldName); } - return false; // TODO: Implementation for Arrow } @Override @@ -284,8 +295,9 @@ public boolean getBoolean(int columnIndex) throws SQLException { } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); return fieldValue.getValue() != null && fieldValue.getBooleanValue(); + } else { // Data received from Read API (Arrow) + return getBoolean(schemaFieldList.get(columnIndex).getName()); } - return false; // TODO: Implementation for Arrow } @Override @@ -298,8 +310,10 @@ public byte[] getBytes(String fieldName) throws SQLException { } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); return fieldValue.getValue() == null ? null : fieldValue.getBytesValue(); + } else { // Data received from Read API (Arrow) + Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; + return (byte[]) curTuple.x().get(fieldName); } - return null; // TODO: Implementation for Arrow } @Override @@ -309,8 +323,9 @@ public byte[] getBytes(int columnIndex) throws SQLException { } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); return fieldValue.getValue() == null ? null : fieldValue.getBytesValue(); + } else { // Data received from Read API (Arrow) + return getBytes(schemaFieldList.get(columnIndex).getName()); } - return null; // TODO: Implementation for Arrow } @Override From 820b454985a0a8126116a3e27e4ccbe02e7a535c Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 23 Feb 2022 21:46:54 +0530 Subject: [PATCH 136/277] Added Arrow specific parsing logic for primitive getters --- .../google/cloud/bigquery/ConnectionImpl.java | 42 ++++++++++++++++--- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 595ccee34..da05812d1 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -60,8 +60,11 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.BitVector; import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.Float8Vector; import org.apache.arrow.vector.TimeStampMicroVector; +import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorLoader; import org.apache.arrow.vector.VectorSchemaRoot; @@ -752,21 +755,48 @@ public void processRows( root.getVector( field.getName()); // can be accessed using the index or Vector/column name // Now cast the FieldVector depending on the type - if (field.getType().getStandardType() == StandardSQLTypeName.STRING) { + if (field.getType().getStandardType() + == StandardSQLTypeName.STRING) { // maps with getString VarCharVector varCharVector = (VarCharVector) curFieldVec; curRow.put( field.getName(), new String(varCharVector.get(rowNum))); // store the row:value mapping - } else if (field.getType().getStandardType() == StandardSQLTypeName.TIMESTAMP) { + } else if (field.getType().getStandardType() + == StandardSQLTypeName.TIMESTAMP) { // maps with getTimeStamp TimeStampMicroVector timeStampMicroVector = (TimeStampMicroVector) curFieldVec; curRow.put( field.getName(), timeStampMicroVector.get(rowNum)); // store the row:value mapping - } else if (field.getType().getStandardType() == StandardSQLTypeName.INT64) { + } else if (field.getType().getStandardType() + == StandardSQLTypeName.INT64) { // maps with getLong and getInt BigIntVector bigIntVector = (BigIntVector) curFieldVec; curRow.put( - field.getName(), (int) bigIntVector.get(rowNum)); // store the row:value mapping + field.getName(), + bigIntVector.get( + rowNum)); // store the row:value mapping. Storing the value as long, it will + // be used as long in getLong method, and it will be casted back to + // int in getInt + } else if (field.getType().getStandardType() == StandardSQLTypeName.FLOAT64 + || field.getType().getStandardType() + == StandardSQLTypeName.NUMERIC) { // maps with getDouble and getBigDecimal + Float8Vector float8Vector = (Float8Vector) curFieldVec; + curRow.put(field.getName(), float8Vector.get(rowNum)); // store the row:value mapping + } else if (field.getType().getStandardType() + == StandardSQLTypeName.BOOL) { // maps with getBoolean + BitVector bitVector = (BitVector) curFieldVec; + curRow.put( + field.getName(), + bitVector.get(rowNum) + == 1); // store the row:value mapping, checking if the bit (it's a 1 bit + // vector) is set using equals + } else if (field.getType().getStandardType() + == StandardSQLTypeName.BYTES) { // maps with getBoolean + VarBinaryVector binaryVector = (VarBinaryVector) curFieldVec; + curRow.put(field.getName(), binaryVector.get(rowNum)); // store the row:value mapping } else { - throw new RuntimeException("TODO: Implement remaining support type conversions"); + throw new RuntimeException( + String.format( + "StandardSQLTypeName %s is not implemented", + field.getType().getStandardType())); } } buffer.put(Tuple.of(curRow, true)); @@ -774,7 +804,7 @@ public void processRows( root.clear(); // TODO: make sure to clear the root while implementing the thread // interruption logic (Connection.close method) - } catch (InterruptedException e) { + } catch (RuntimeException | InterruptedException e) { throw BigQueryException.translateAndThrow(e); } finally { try { From 661dbf56314fcef10b6692bf9d76f40e106b669a Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 24 Feb 2022 12:43:57 +0530 Subject: [PATCH 137/277] Added Arrow specific parsing logic for additional datatypes --- .../google/cloud/bigquery/ConnectionImpl.java | 38 ++++++++++++++++--- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index da05812d1..61841283a 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -44,6 +44,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import java.io.IOException; +import java.sql.Timestamp; import java.util.AbstractList; import java.util.ArrayList; import java.util.HashMap; @@ -61,8 +62,10 @@ import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; +import org.apache.arrow.vector.DateMilliVector; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.TimeMilliVector; import org.apache.arrow.vector.TimeStampMicroVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; @@ -755,8 +758,10 @@ public void processRows( root.getVector( field.getName()); // can be accessed using the index or Vector/column name // Now cast the FieldVector depending on the type - if (field.getType().getStandardType() - == StandardSQLTypeName.STRING) { // maps with getString + if (field.getType().getStandardType() == StandardSQLTypeName.STRING + || field.getType().getStandardType() + == StandardSQLTypeName + .GEOGRAPHY) { // maps with getString (for both String and Geography) VarCharVector varCharVector = (VarCharVector) curFieldVec; curRow.put( field.getName(), @@ -773,8 +778,8 @@ public void processRows( field.getName(), bigIntVector.get( rowNum)); // store the row:value mapping. Storing the value as long, it will - // be used as long in getLong method, and it will be casted back to - // int in getInt + // be used as long in getLong method, and it will be casted back to + // int in getInt } else if (field.getType().getStandardType() == StandardSQLTypeName.FLOAT64 || field.getType().getStandardType() == StandardSQLTypeName.NUMERIC) { // maps with getDouble and getBigDecimal @@ -787,11 +792,34 @@ public void processRows( field.getName(), bitVector.get(rowNum) == 1); // store the row:value mapping, checking if the bit (it's a 1 bit - // vector) is set using equals + // vector) is set using equals } else if (field.getType().getStandardType() == StandardSQLTypeName.BYTES) { // maps with getBoolean VarBinaryVector binaryVector = (VarBinaryVector) curFieldVec; curRow.put(field.getName(), binaryVector.get(rowNum)); // store the row:value mapping + } else if (field.getType().getStandardType() + == StandardSQLTypeName + .DATETIME) { // maps with getString (Converting DATETIME back to String) + DateMilliVector dateMilliVector = (DateMilliVector) curFieldVec; + Timestamp dateTime = new Timestamp(dateMilliVector.get(rowNum)); + curRow.put(field.getName(), dateTime.toString()); // store the row:value mapping + } else if (field.getType().getStandardType() + == StandardSQLTypeName.TIME) { // maps with getTime + TimeMilliVector timeMilliVector = (TimeMilliVector) curFieldVec; + curRow.put( + field.getName(), timeMilliVector.get(rowNum)); // store the row:value mapping + } else if (field.getType().getStandardType() + == StandardSQLTypeName.STRUCT) { // maps with getString and getObject + curRow.put( + field.getName(), + curFieldVec.getObject( + rowNum)); // TODO(prasmish): Check how to convert the object back to string + } else if (field.getType().getStandardType() + == StandardSQLTypeName.ARRAY) { // maps with getString + curRow.put( + field.getName(), + curFieldVec.getObject( + rowNum)); // TODO(prasmish): Check how to convert the object back to string } else { throw new RuntimeException( String.format( From b4cfb2d790d2cc8b02ef41a5ecbda0207b5d37ba Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 24 Feb 2022 12:44:50 +0530 Subject: [PATCH 138/277] Modified the getter's logic for parsing arrow --- .../cloud/bigquery/BigQueryResultSetImpl.java | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index fba0229e6..6b347ab97 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -103,8 +103,10 @@ public Object getObject(String fieldName) throws SQLException { } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); return fieldValue.getValue() == null ? null : fieldValue.getValue(); + } else { // Data received from Read API (Arrow) + Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; + return curTuple.x().get(fieldName); } - return null; // TODO: Implementation for Arrow } @Override @@ -114,8 +116,9 @@ public Object getObject(int columnIndex) throws SQLException { } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); return fieldValue.getValue() == null ? null : fieldValue.getValue(); + } else { // Data received from Read API (Arrow) + return getObject(schemaFieldList.get(columnIndex).getName()); } - return null; // TODO: Implementation for Arrow } @Override @@ -130,7 +133,12 @@ public String getString(String fieldName) throws SQLException { return fieldValue.getValue() == null ? null : fieldValue.getStringValue(); } else { // Data received from Read API (Arrow) Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; - return (String) curTuple.x().get(fieldName); + Object currentVal = curTuple.x().get(fieldName); + if (currentVal instanceof String) { + return (String) curTuple.x().get(fieldName); + } else { + return curTuple.x().get(fieldName).toString(); + } } } @@ -362,7 +370,7 @@ public Timestamp getTimestamp(int columnIndex) throws SQLException { fieldValue.getTimestampValue() / 1000); // getTimestampValue returns time in microseconds, and TimeStamp // expects it in millis - } else { + } else { // Data received from Read API (Arrow) return getTimestamp(schemaFieldList.get(columnIndex).getName()); } } @@ -377,8 +385,11 @@ public Time getTime(String fieldName) throws SQLException { } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); return getTime(fieldValue); + } else { // Data received from Read API (Arrow) + Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; + Integer timeInMilli = (Integer) curTuple.x().get(fieldName); + return new Time(timeInMilli); } - return null; // TODO: Implementation for Arrow } @Override @@ -388,8 +399,9 @@ public Time getTime(int columnIndex) throws SQLException { } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); return getTime(fieldValue); + } else { // Data received from Read API (Arrow) + return getTime(schemaFieldList.get(columnIndex).getName()); } - return null; // TODO: Implementation for Arrow } private Time getTime(FieldValue fieldValue) throws SQLException { From 31910055688c3d7257ad3ba9879c622b8a61f343 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 24 Feb 2022 20:08:40 +0530 Subject: [PATCH 139/277] Added Interrupt logic for Arrow processing --- .../google/cloud/bigquery/ConnectionImpl.java | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 61841283a..3ebb23c70 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -482,8 +482,9 @@ void populateBufferAsync( new EndOfFieldValueList()); // All the pages has been processed, put this marker } catch (InterruptedException e) { throw new BigQueryException(0, e.getMessage(), e); + } finally { + queryTaskExecutor.shutdownNow(); // Shutdown the thread pool } - queryTaskExecutor.shutdownNow(); // Shutdown the thread pool }; queryTaskExecutor.execute(populateBufferRunnable); @@ -686,11 +687,22 @@ private void processArrowStreamAsync( com.google.api.gax.rpc.ServerStream stream = bqReadClient.readRowsCallable().call(readRowsRequest); for (ReadRowsResponse response : stream) { + if (Thread.currentThread().isInterrupted() + || queryTaskExecutor.isShutdown()) { // do not process and shutdown + break; + } reader.processRows(response.getArrowRecordBatch(), buffer, schema); } - buffer.put(Tuple.of(null, false)); // marking end of stream + } catch (Exception e) { throw BigQueryException.translateAndThrow(e); + } finally { + try { + buffer.put(Tuple.of(null, false)); // marking end of stream + queryTaskExecutor.shutdownNow(); // Shutdown the thread pool + } catch (InterruptedException e) { + logger.log(Level.WARNING, "\n Error occurred ", e); + } } }; @@ -751,6 +763,11 @@ public void processRows( rowNum < root.getRowCount(); rowNum++) { // for the given number of rows in the batch + if (Thread.currentThread().isInterrupted() + || queryTaskExecutor.isShutdown()) { // do not process and shutdown + break; // exit the loop, root will be cleared in the finally block + } + Map curRow = new HashMap<>(); for (int col = 0; col < fields.size(); col++) { // iterate all the vectors for a given row com.google.cloud.bigquery.Field field = fields.get(col); From 92a7b59af1fb75844d4311b205ffe0847efba4e8 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 3 Mar 2022 14:28:29 +0530 Subject: [PATCH 140/277] Added Arrow Support in ResultSet Methods --- .../cloud/bigquery/BigQueryResultSetImpl.java | 86 +++++++++++++++---- 1 file changed, 67 insertions(+), 19 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index 6b347ab97..600cd201a 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -26,6 +26,7 @@ import java.time.LocalTime; import java.util.Map; import java.util.concurrent.BlockingQueue; +import org.apache.arrow.vector.util.Text; // TODO: This implementation deals with the JSON response. We can have respective implementations public class BigQueryResultSetImpl implements BigQueryResultSet { @@ -105,6 +106,9 @@ public Object getObject(String fieldName) throws SQLException { return fieldValue.getValue() == null ? null : fieldValue.getValue(); } else { // Data received from Read API (Arrow) Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; + if (!curTuple.x().containsKey(fieldName)) { + throw new SQLException(String.format("Field %s not found", fieldName)); + } return curTuple.x().get(fieldName); } } @@ -133,12 +137,15 @@ public String getString(String fieldName) throws SQLException { return fieldValue.getValue() == null ? null : fieldValue.getStringValue(); } else { // Data received from Read API (Arrow) Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; + if (!curTuple.x().containsKey(fieldName)) { + throw new SQLException(String.format("Field %s not found", fieldName)); + } Object currentVal = curTuple.x().get(fieldName); - if (currentVal instanceof String) { - return (String) curTuple.x().get(fieldName); - } else { - return curTuple.x().get(fieldName).toString(); + if (currentVal == null) { + return null; } + Text textVal = (Text) currentVal; + return textVal.toString(); } } @@ -166,9 +173,13 @@ public int getInt(String fieldName) throws SQLException { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); return fieldValue.getValue() == null ? 0 : fieldValue.getNumericValue().intValue(); } else { // Data received from Read API (Arrow) + Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; - return ((Long) curTuple.x().get(fieldName)) - .intValue(); // BigIntVector.get(...) returns Long + if (!curTuple.x().containsKey(fieldName)) { + throw new SQLException(String.format("Field %s not found", fieldName)); + } + Object curVal = curTuple.x().get(fieldName); + return curVal == null ? 0 : ((BigDecimal) curVal).intValue(); } } @@ -198,7 +209,11 @@ public long getLong(String fieldName) throws SQLException { return fieldValue.getValue() == null ? 0L : fieldValue.getNumericValue().longValue(); } else { // Data received from Read API (Arrow) Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; - return (Long) curTuple.x().get(fieldName); // BigIntVector.get(...) returns Long + if (!curTuple.x().containsKey(fieldName)) { + throw new SQLException(String.format("Field %s not found", fieldName)); + } + Object curVal = curTuple.x().get(fieldName); + return curVal == null ? 0 : ((BigDecimal) curVal).longValue(); } } @@ -228,7 +243,11 @@ public double getDouble(String fieldName) throws SQLException { return fieldValue.getValue() == null ? 0d : fieldValue.getNumericValue().doubleValue(); } else { // Data received from Read API (Arrow) Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; - return (Double) curTuple.x().get(fieldName); + if (!curTuple.x().containsKey(fieldName)) { + throw new SQLException(String.format("Field %s not found", fieldName)); + } + Object curVal = curTuple.x().get(fieldName); + return curVal == null ? 0.0d : ((BigDecimal) curVal).doubleValue(); } } @@ -292,7 +311,11 @@ public boolean getBoolean(String fieldName) throws SQLException { return fieldValue.getValue() != null && fieldValue.getBooleanValue(); } else { // Data received from Read API (Arrow) Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; - return (Boolean) curTuple.x().get(fieldName); + if (!curTuple.x().containsKey(fieldName)) { + throw new SQLException(String.format("Field %s not found", fieldName)); + } + Object curVal = curTuple.x().get(fieldName); + return curVal != null && (Boolean) curVal; } } @@ -320,7 +343,11 @@ public byte[] getBytes(String fieldName) throws SQLException { return fieldValue.getValue() == null ? null : fieldValue.getBytesValue(); } else { // Data received from Read API (Arrow) Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; - return (byte[]) curTuple.x().get(fieldName); + if (!curTuple.x().containsKey(fieldName)) { + throw new SQLException(String.format("Field %s not found", fieldName)); + } + Object curVal = curTuple.x().get(fieldName); + return curVal == null ? null : (byte[]) curVal; } } @@ -353,8 +380,13 @@ public Timestamp getTimestamp(String fieldName) throws SQLException { // expects it in millis } else { Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; - return new Timestamp( - (Long) curTuple.x().get(fieldName) / 1000); // Timestamp is represented as a Long + if (!curTuple.x().containsKey(fieldName)) { + throw new SQLException(String.format("Field %s not found", fieldName)); + } + Object timeStampVal = curTuple.x().get(fieldName); + return timeStampVal == null + ? null + : new Timestamp((Long) timeStampVal / 1000); // Timestamp is represented as a Long } } @@ -384,11 +416,15 @@ public Time getTime(String fieldName) throws SQLException { return null; // if the value is SQL NULL, the value returned is null } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); - return getTime(fieldValue); + return getTimeFromFieldVal(fieldValue); } else { // Data received from Read API (Arrow) Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; - Integer timeInMilli = (Integer) curTuple.x().get(fieldName); - return new Time(timeInMilli); + if (!curTuple.x().containsKey(fieldName)) { + throw new SQLException(String.format("Field %s not found", fieldName)); + } + + Object timeStampObj = curTuple.x().get(fieldName); + return timeStampObj == null ? null : new Time(((Long) timeStampObj).intValue()); } } @@ -398,13 +434,13 @@ public Time getTime(int columnIndex) throws SQLException { return null; // if the value is SQL NULL, the value returned is null } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); - return getTime(fieldValue); + return getTimeFromFieldVal(fieldValue); } else { // Data received from Read API (Arrow) return getTime(schemaFieldList.get(columnIndex).getName()); } } - private Time getTime(FieldValue fieldValue) throws SQLException { + private Time getTimeFromFieldVal(FieldValue fieldValue) throws SQLException { if (fieldValue.getValue() != null) { // Time ranges from 00:00:00 to 23:59:59.99999. in BigQuery. Parsing it to java.sql.Time String strTime = fieldValue.getStringValue(); @@ -438,8 +474,19 @@ public Date getDate(String fieldName) throws SQLException { } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); return fieldValue.getValue() == null ? null : Date.valueOf(fieldValue.getStringValue()); + } else { // Data received from Read API (Arrow) + Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; + if (!curTuple.x().containsKey(fieldName)) { + throw new SQLException(String.format("Field %s not found", fieldName)); + } + Object dateObj = curTuple.x().get(fieldName); + if (dateObj == null) { + return null; + } else { + Integer dateInt = (Integer) dateObj; + return new Date(dateInt); + } } - return null; // TODO: Implementation for Arrow } @Override @@ -449,8 +496,9 @@ public Date getDate(int columnIndex) throws SQLException { } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); return fieldValue.getValue() == null ? null : Date.valueOf(fieldValue.getStringValue()); + } else { // Data received from Read API (Arrow) + return getDate(schemaFieldList.get(columnIndex).getName()); } - return null; // TODO: Implementation for Arrow } } From 77db382312e6d09ae88d9932e744afcb2cea4477 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 3 Mar 2022 14:32:10 +0530 Subject: [PATCH 141/277] Removed typecasting from processRows method --- .../google/cloud/bigquery/ConnectionImpl.java | 80 +------------------ 1 file changed, 2 insertions(+), 78 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 3ebb23c70..4cde986ee 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -44,7 +44,6 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import java.io.IOException; -import java.sql.Timestamp; import java.util.AbstractList; import java.util.ArrayList; import java.util.HashMap; @@ -60,15 +59,7 @@ import java.util.stream.Collectors; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; -import org.apache.arrow.vector.BigIntVector; -import org.apache.arrow.vector.BitVector; -import org.apache.arrow.vector.DateMilliVector; import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.Float8Vector; -import org.apache.arrow.vector.TimeMilliVector; -import org.apache.arrow.vector.TimeStampMicroVector; -import org.apache.arrow.vector.VarBinaryVector; -import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorLoader; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.ipc.ReadChannel; @@ -661,6 +652,7 @@ private BigQueryResultSet highThroughPutRead( new ArrowRowReader(readSession.getArrowSchema(), arrowNameToIndex), schema); + logger.log(Level.INFO, "\n Using BigQuery Read API"); return new BigQueryResultSetImpl, Boolean>>( schema, totalRows, buffer, stats); @@ -774,75 +766,7 @@ public void processRows( FieldVector curFieldVec = root.getVector( field.getName()); // can be accessed using the index or Vector/column name - // Now cast the FieldVector depending on the type - if (field.getType().getStandardType() == StandardSQLTypeName.STRING - || field.getType().getStandardType() - == StandardSQLTypeName - .GEOGRAPHY) { // maps with getString (for both String and Geography) - VarCharVector varCharVector = (VarCharVector) curFieldVec; - curRow.put( - field.getName(), - new String(varCharVector.get(rowNum))); // store the row:value mapping - } else if (field.getType().getStandardType() - == StandardSQLTypeName.TIMESTAMP) { // maps with getTimeStamp - TimeStampMicroVector timeStampMicroVector = (TimeStampMicroVector) curFieldVec; - curRow.put( - field.getName(), timeStampMicroVector.get(rowNum)); // store the row:value mapping - } else if (field.getType().getStandardType() - == StandardSQLTypeName.INT64) { // maps with getLong and getInt - BigIntVector bigIntVector = (BigIntVector) curFieldVec; - curRow.put( - field.getName(), - bigIntVector.get( - rowNum)); // store the row:value mapping. Storing the value as long, it will - // be used as long in getLong method, and it will be casted back to - // int in getInt - } else if (field.getType().getStandardType() == StandardSQLTypeName.FLOAT64 - || field.getType().getStandardType() - == StandardSQLTypeName.NUMERIC) { // maps with getDouble and getBigDecimal - Float8Vector float8Vector = (Float8Vector) curFieldVec; - curRow.put(field.getName(), float8Vector.get(rowNum)); // store the row:value mapping - } else if (field.getType().getStandardType() - == StandardSQLTypeName.BOOL) { // maps with getBoolean - BitVector bitVector = (BitVector) curFieldVec; - curRow.put( - field.getName(), - bitVector.get(rowNum) - == 1); // store the row:value mapping, checking if the bit (it's a 1 bit - // vector) is set using equals - } else if (field.getType().getStandardType() - == StandardSQLTypeName.BYTES) { // maps with getBoolean - VarBinaryVector binaryVector = (VarBinaryVector) curFieldVec; - curRow.put(field.getName(), binaryVector.get(rowNum)); // store the row:value mapping - } else if (field.getType().getStandardType() - == StandardSQLTypeName - .DATETIME) { // maps with getString (Converting DATETIME back to String) - DateMilliVector dateMilliVector = (DateMilliVector) curFieldVec; - Timestamp dateTime = new Timestamp(dateMilliVector.get(rowNum)); - curRow.put(field.getName(), dateTime.toString()); // store the row:value mapping - } else if (field.getType().getStandardType() - == StandardSQLTypeName.TIME) { // maps with getTime - TimeMilliVector timeMilliVector = (TimeMilliVector) curFieldVec; - curRow.put( - field.getName(), timeMilliVector.get(rowNum)); // store the row:value mapping - } else if (field.getType().getStandardType() - == StandardSQLTypeName.STRUCT) { // maps with getString and getObject - curRow.put( - field.getName(), - curFieldVec.getObject( - rowNum)); // TODO(prasmish): Check how to convert the object back to string - } else if (field.getType().getStandardType() - == StandardSQLTypeName.ARRAY) { // maps with getString - curRow.put( - field.getName(), - curFieldVec.getObject( - rowNum)); // TODO(prasmish): Check how to convert the object back to string - } else { - throw new RuntimeException( - String.format( - "StandardSQLTypeName %s is not implemented", - field.getType().getStandardType())); - } + curRow.put(field.getName(), curFieldVec.getObject(rowNum)); // Added the raw value } buffer.put(Tuple.of(curRow, true)); } From 1f64c639bf89e1a9ce9db4b1475a6c2d3f9549d6 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 3 Mar 2022 19:44:55 +0530 Subject: [PATCH 142/277] Added Json typecasting logic --- .../google/cloud/bigquery/BigQueryResultSetImpl.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index 600cd201a..ce345fc2e 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -26,6 +26,7 @@ import java.time.LocalTime; import java.util.Map; import java.util.concurrent.BlockingQueue; +import org.apache.arrow.vector.util.JsonStringArrayList; import org.apache.arrow.vector.util.Text; // TODO: This implementation deals with the JSON response. We can have respective implementations @@ -144,8 +145,13 @@ public String getString(String fieldName) throws SQLException { if (currentVal == null) { return null; } - Text textVal = (Text) currentVal; - return textVal.toString(); + if (currentVal instanceof JsonStringArrayList) { // arrays + JsonStringArrayList jsnAry = (JsonStringArrayList) currentVal; + return jsnAry.toString(); + } else { + Text textVal = (Text) currentVal; + return textVal.toString(); + } } } From d7cbe092b33bba8b752a137c03d1c877ddbca8c3 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 4 Mar 2022 14:27:38 +0530 Subject: [PATCH 143/277] Updated Arrow dependency to latest --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 22e447754..9b8175e9a 100644 --- a/pom.xml +++ b/pom.xml @@ -174,12 +174,12 @@ org.apache.arrow arrow-vector - 6.0.0 + 7.0.0 org.apache.arrow arrow-memory-core - 6.0.0 + 7.0.0 org.apache.arrow From 2c01cf6a951520373d8ef286d55a3ed3a0fa0166 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 4 Mar 2022 14:48:45 +0530 Subject: [PATCH 144/277] Added UseReadAPI property --- .../cloud/bigquery/ConnectionSettings.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java index cdc9dbcc8..2b1821f4a 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java @@ -32,6 +32,13 @@ public abstract class ConnectionSettings { // Package private so users can't subclass it but AutoValue can. } + /** + * Returns useReadAPI flag, enabled by default. Read API will be used if the underlying conditions + * are satisfied and this flag is enabled + */ + @Nullable + public abstract Boolean getUseReadAPI(); + /** Returns the synchronous response timeoutMs associated with this query */ @Nullable public abstract Long getRequestTimeout(); @@ -189,6 +196,15 @@ public static Builder newBuilder() { @AutoValue.Builder public abstract static class Builder { + /** + * Sets useReadAPI flag, enabled by default. Read API will be used if the underlying conditions + * are satisfied and this flag is enabled + * + * @param useReadAPI or {@code true} for none + */ + @Nullable + public abstract Builder setUseReadAPI(Boolean useReadAPI); + /** * Sets how long to wait for the query to complete, in milliseconds, before the request times * out and returns. Note that this is only a timeout for the request, not the query. If the From 999df03a6f707abe649cc81874e29d36cbd3546d Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 4 Mar 2022 14:52:34 +0530 Subject: [PATCH 145/277] Updated useReadAPI --- .../main/java/com/google/cloud/bigquery/ConnectionImpl.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 4cde986ee..d06217c46 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -849,8 +849,9 @@ boolean isFastQuerySupported() { private boolean useReadAPI(Long totalRows, Long pageRows) { long resultRatio = totalRows / pageRows; - if (connectionSettings.getReadClientConnectionConfiguration() - != null) { // Adding a null check to avoid NPE + if (connectionSettings.getUseReadAPI() + && connectionSettings.getReadClientConnectionConfiguration() + != null) { // Adding a null check to avoid NPE return resultRatio > connectionSettings .getReadClientConnectionConfiguration() From b9bdd852c1a4cfe4e83588bc57e39f837628913d Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 4 Mar 2022 14:56:52 +0530 Subject: [PATCH 146/277] Updated useReadAPI --- .../src/main/java/com/google/cloud/bigquery/ConnectionImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index d06217c46..d491dc738 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -849,7 +849,7 @@ boolean isFastQuerySupported() { private boolean useReadAPI(Long totalRows, Long pageRows) { long resultRatio = totalRows / pageRows; - if (connectionSettings.getUseReadAPI() + if (Boolean.TRUE.equals(connectionSettings.getUseReadAPI()) && connectionSettings.getReadClientConnectionConfiguration() != null) { // Adding a null check to avoid NPE return resultRatio From 3986358e6a8b0a7c18f4240b21fddeab82c2f66e Mon Sep 17 00:00:00 2001 From: Stephanie Wang Date: Sat, 12 Feb 2022 02:17:33 +0530 Subject: [PATCH 147/277] feat: add Interval type support (#1844) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add Interval type support Fixes b/208051516 * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * add threeten-extra PeriodDuration support * add unit test coverage Co-authored-by: Owl Bot --- google-cloud-bigquery/pom.xml | 4 + .../cloud/bigquery/LegacySQLTypeName.java | 3 + .../cloud/bigquery/QueryParameterValue.java | 20 ++++- .../cloud/bigquery/StandardSQLTypeName.java | 6 +- .../bigquery/QueryParameterValueTest.java | 20 +++++ .../cloud/bigquery/it/ITBigQueryTest.java | 73 +++++++++++++++++++ pom.xml | 8 ++ 7 files changed, 131 insertions(+), 3 deletions(-) diff --git a/google-cloud-bigquery/pom.xml b/google-cloud-bigquery/pom.xml index 33690ffc9..0fa6cf449 100644 --- a/google-cloud-bigquery/pom.xml +++ b/google-cloud-bigquery/pom.xml @@ -85,6 +85,10 @@ com.google.code.gson gson + + org.threeten + threeten-extra + diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LegacySQLTypeName.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LegacySQLTypeName.java index 1c2a3d884..944df2fb0 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LegacySQLTypeName.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LegacySQLTypeName.java @@ -96,6 +96,9 @@ public LegacySQLTypeName apply(String constant) { /** Represents JSON data */ public static final LegacySQLTypeName JSON = type.createAndRegister("JSON").setStandardType(StandardSQLTypeName.JSON); + /** Represents duration or amount of time. */ + public static final LegacySQLTypeName INTERVAL = + type.createAndRegister("INTERVAL").setStandardType(StandardSQLTypeName.INTERVAL); private static Map standardToLegacyMap = new HashMap<>(); diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryParameterValue.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryParameterValue.java index f7f7e384f..227597bb1 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryParameterValue.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryParameterValue.java @@ -43,6 +43,7 @@ import org.threeten.bp.format.DateTimeFormatter; import org.threeten.bp.format.DateTimeFormatterBuilder; import org.threeten.bp.format.DateTimeParseException; +import org.threeten.extra.PeriodDuration; /** * A value for a QueryParameter along with its type. @@ -63,6 +64,7 @@ *

  • BigDecimal: StandardSQLTypeName.NUMERIC *
  • BigNumeric: StandardSQLTypeName.BIGNUMERIC *
  • JSON: StandardSQLTypeName.JSON + *
  • INTERVAL: StandardSQLTypeName.INTERVAL * * *

    No other types are supported through that entry point. The other types can be created by @@ -308,12 +310,26 @@ public static QueryParameterValue time(String value) { /** * Creates a {@code QueryParameterValue} object with a type of DATETIME. Must be in the format - * "yyyy-MM-dd HH:mm:ss.SSSSSS", e.g. ""2014-08-19 12:41:35.220000". + * "yyyy-MM-dd HH:mm:ss.SSSSSS", e.g. "2014-08-19 12:41:35.220000". */ public static QueryParameterValue dateTime(String value) { return of(value, StandardSQLTypeName.DATETIME); } + /** + * Creates a {@code QueryParameterValue} object with a type of INTERVAL. Must be in the canonical + * format "[sign]Y-M [sign]D [sign]H:M:S[.F]", e.g. "123-7 -19 0:24:12.000006" or ISO 8601 + * duration format, e.g. "P123Y7M-19DT0H24M12.000006S" + */ + public static QueryParameterValue interval(String value) { + return of(value, StandardSQLTypeName.INTERVAL); + } + + /** Creates a {@code QueryParameterValue} object with a type of INTERVAL. */ + public static QueryParameterValue interval(PeriodDuration value) { + return of(value, StandardSQLTypeName.INTERVAL); + } + /** * Creates a {@code QueryParameterValue} object with a type of ARRAY, and an array element type * based on the given class. @@ -408,6 +424,8 @@ private static String valueToStringOrNull(T value, StandardSQLTypeName type) return value.toString(); case JSON: if (value instanceof String || value instanceof JsonObject) return value.toString(); + case INTERVAL: + if (value instanceof String || value instanceof PeriodDuration) return value.toString(); break; case STRUCT: throw new IllegalArgumentException("Cannot convert STRUCT to String value"); diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardSQLTypeName.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardSQLTypeName.java index 57152a2a6..1081fc5d6 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardSQLTypeName.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardSQLTypeName.java @@ -57,6 +57,8 @@ public enum StandardSQLTypeName { DATETIME, /** Represents a set of geographic points, represented as a Well Known Text (WKT) string. */ GEOGRAPHY, - /** Represents JSON data */ - JSON + /** Represents JSON data. */ + JSON, + /** Represents duration or amount of time. */ + INTERVAL } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryParameterValueTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryParameterValueTest.java index 679b6ec5c..6b5368003 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryParameterValueTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryParameterValueTest.java @@ -27,6 +27,7 @@ import com.google.gson.JsonObject; import java.math.BigDecimal; import java.text.ParseException; +import java.time.Period; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -38,6 +39,7 @@ import org.threeten.bp.format.DateTimeFormatter; import org.threeten.bp.format.DateTimeFormatterBuilder; import org.threeten.bp.jdk8.Jdk8Methods; +import org.threeten.extra.PeriodDuration; public class QueryParameterValueTest { @@ -212,6 +214,24 @@ public void testJson() { assertThat(value1.getArrayType()).isNull(); } + @Test + public void testInterval() { + QueryParameterValue value = QueryParameterValue.interval("123-7 -19 0:24:12.000006"); + QueryParameterValue value1 = QueryParameterValue.interval("P123Y7M-19DT0H24M12.000006S"); + QueryParameterValue value2 = + QueryParameterValue.interval( + PeriodDuration.of(Period.of(1, 2, 25), java.time.Duration.ofHours(8))); + assertThat(value.getValue()).isEqualTo("123-7 -19 0:24:12.000006"); + assertThat(value1.getValue()).isEqualTo("P123Y7M-19DT0H24M12.000006S"); + assertThat(value2.getValue()).isEqualTo("P1Y2M25DT8H"); + assertThat(value.getType()).isEqualTo(StandardSQLTypeName.INTERVAL); + assertThat(value1.getType()).isEqualTo(StandardSQLTypeName.INTERVAL); + assertThat(value2.getType()).isEqualTo(StandardSQLTypeName.INTERVAL); + assertThat(value.getArrayType()).isNull(); + assertThat(value1.getArrayType()).isNull(); + assertThat(value2.getArrayType()).isNull(); + } + @Test public void testBytes() { QueryParameterValue value = QueryParameterValue.bytes(new byte[] {1, 3}); diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index b59772888..fc0c2a847 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -141,6 +141,7 @@ import java.sql.SQLException; import java.sql.Time; import java.time.Instant; +import java.time.Period; import java.time.LocalTime; import java.util.ArrayList; import java.util.Collection; @@ -163,6 +164,7 @@ import org.junit.Test; import org.junit.rules.Timeout; import org.threeten.bp.Duration; +import org.threeten.extra.PeriodDuration; public class ITBigQueryTest { @@ -1054,6 +1056,77 @@ public void testJsonType() throws InterruptedException { } } + @Test + public void testIntervalType() throws InterruptedException { + String tableName = "test_create_table_intervaltype"; + TableId tableId = TableId.of(DATASET, tableName); + Schema schema = Schema.of(Field.of("intervalField", StandardSQLTypeName.INTERVAL)); + StandardTableDefinition standardTableDefinition = StandardTableDefinition.of(schema); + try { + // Create a table with a JSON column + Table createdTable = bigquery.create(TableInfo.of(tableId, standardTableDefinition)); + assertNotNull(createdTable); + + // Insert 3 rows of Interval data into the Interval column + Map intervalRow1 = + Collections.singletonMap("intervalField", "123-7 -19 0:24:12.000006"); + Map intervalRow2 = + Collections.singletonMap("intervalField", "P123Y7M-19DT0H24M12.000006S"); + + InsertAllRequest request = + InsertAllRequest.newBuilder(tableId).addRow(intervalRow1).addRow(intervalRow2).build(); + InsertAllResponse response = bigquery.insertAll(request); + assertFalse(response.hasErrors()); + assertEquals(0, response.getInsertErrors().size()); + + // Insert another Interval row parsed from a String with Interval positional query parameter + String dml = "INSERT INTO " + tableId.getTable() + " (intervalField) VALUES(?)"; + // Parsing from ISO 8610 format String + QueryParameterValue intervalParameter = + QueryParameterValue.interval("P125Y7M-19DT0H24M12.000006S"); + QueryJobConfiguration dmlQueryJobConfiguration = + QueryJobConfiguration.newBuilder(dml) + .setDefaultDataset(DatasetId.of(DATASET)) + .setUseLegacySql(false) + .addPositionalParameter(intervalParameter) + .build(); + bigquery.query(dmlQueryJobConfiguration); + Page rows = bigquery.listTableData(tableId); + assertEquals(3, Iterables.size(rows.getValues())); + + // Parsing from threeten-extra PeriodDuration + QueryParameterValue intervalParameter1 = + QueryParameterValue.interval( + PeriodDuration.of(Period.of(1, 2, 25), java.time.Duration.ofHours(8))); + QueryJobConfiguration dmlQueryJobConfiguration1 = + QueryJobConfiguration.newBuilder(dml) + .setDefaultDataset(DatasetId.of(DATASET)) + .setUseLegacySql(false) + .addPositionalParameter(intervalParameter1) + .build(); + bigquery.query(dmlQueryJobConfiguration1); + Page rows1 = bigquery.listTableData(tableId); + assertEquals(4, Iterables.size(rows1.getValues())); + + // Query the Interval column with Interval positional query parameter + String sql = "SELECT intervalField FROM " + tableId.getTable() + " WHERE intervalField = ? "; + QueryParameterValue intervalParameter2 = + QueryParameterValue.interval("P125Y7M-19DT0H24M12.000006S"); + QueryJobConfiguration queryJobConfiguration = + QueryJobConfiguration.newBuilder(sql) + .setDefaultDataset(DatasetId.of(DATASET)) + .setUseLegacySql(false) + .addPositionalParameter(intervalParameter2) + .build(); + TableResult result = bigquery.query(queryJobConfiguration); + for (FieldValueList values : result.iterateAll()) { + assertEquals("125-7 -19 0:24:12.000006", values.get(0).getValue()); + } + } finally { + assertTrue(bigquery.delete(tableId)); + } + } + @Test public void testCreateTableWithConstraints() { String tableName = "test_create_table_with_constraints"; diff --git a/pom.xml b/pom.xml index 9b8175e9a..271bc0db5 100644 --- a/pom.xml +++ b/pom.xml @@ -108,12 +108,20 @@ ${google-api-services-bigquery.version} + com.google.code.gson gson 2.8.9 + + + org.threeten + threeten-extra + 1.7.0 + + junit From 4248a7f2901b9dc8b89600667c7f5251f1ca2f55 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Mon, 7 Mar 2022 12:06:16 +0530 Subject: [PATCH 148/277] Reformat --- .../test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index fc0c2a847..a1b6f6e61 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -141,8 +141,8 @@ import java.sql.SQLException; import java.sql.Time; import java.time.Instant; -import java.time.Period; import java.time.LocalTime; +import java.time.Period; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; From 87e3f9e672371d510321293c76929eee40329a0c Mon Sep 17 00:00:00 2001 From: stephwang Date: Mon, 7 Mar 2022 13:02:25 -0500 Subject: [PATCH 149/277] remove unused declacred dependency: org.apache.arrow:arrow-memory-netty:jar:7.0.0:runtime --- pom.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pom.xml b/pom.xml index 35452e419..c29606570 100644 --- a/pom.xml +++ b/pom.xml @@ -196,12 +196,6 @@ arrow-memory-core 7.0.0 - - org.apache.arrow - arrow-memory-netty - 7.0.0 - runtime - From b7076e712a4f9a531cc19fab46567e3418f10c81 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Mon, 7 Mar 2022 22:32:30 +0000 Subject: [PATCH 150/277] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot?= =?UTF-8?q?=20post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8b986aae0..a754bfc97 100644 --- a/README.md +++ b/README.md @@ -59,13 +59,13 @@ implementation 'com.google.cloud:google-cloud-bigquery' If you are using Gradle without BOM, add this to your dependencies ```Groovy -implementation 'com.google.cloud:google-cloud-bigquery:2.9.1' +implementation 'com.google.cloud:google-cloud-bigquery:2.9.2' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.9.1" +libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.9.2" ``` ## Authentication From 45440de4f22790119e371435d6a00fc23a317e2a Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 8 Mar 2022 10:26:16 +0530 Subject: [PATCH 151/277] fixed getDate --- .../com/google/cloud/bigquery/BigQueryResultSetImpl.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index ce345fc2e..04bcff983 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -490,7 +490,12 @@ public Date getDate(String fieldName) throws SQLException { return null; } else { Integer dateInt = (Integer) dateObj; - return new Date(dateInt); + long dateInMillis = + Long.valueOf(dateInt) + * (24 * 60 * 60 + * 1000); // For example int 18993 represents 2022-01-01, converting time to + // milli seconds + return new Date(dateInMillis); } } } From d77bb27d8f98170e3609aa10f2c8d8fb8eb2b524 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 8 Mar 2022 15:15:08 +0530 Subject: [PATCH 152/277] format --- .../java/com/google/cloud/bigquery/BigQueryResultSetImpl.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index 04bcff983..ffa91c299 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -428,7 +428,6 @@ public Time getTime(String fieldName) throws SQLException { if (!curTuple.x().containsKey(fieldName)) { throw new SQLException(String.format("Field %s not found", fieldName)); } - Object timeStampObj = curTuple.x().get(fieldName); return timeStampObj == null ? null : new Time(((Long) timeStampObj).intValue()); } @@ -494,7 +493,7 @@ public Date getDate(String fieldName) throws SQLException { Long.valueOf(dateInt) * (24 * 60 * 60 * 1000); // For example int 18993 represents 2022-01-01, converting time to - // milli seconds + // milli seconds return new Date(dateInMillis); } } From a8b63d25e8c06af7a2af94a3d2c1581953abef74 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 8 Mar 2022 15:16:10 +0530 Subject: [PATCH 153/277] Added containsIntervalType check --- .../google/cloud/bigquery/ConnectionImpl.java | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index d491dc738..48c79eefc 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -47,8 +47,10 @@ import java.util.AbstractList; import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Queue; import java.util.UUID; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; @@ -532,7 +534,7 @@ private int getPageCacheSize(Long numBufferedRows, long numRows, Schema schema) BigQueryResultSet getSubsequentQueryResultsWithJob( Long totalRows, Long pageRows, JobId jobId, GetQueryResultsResponse firstPage) { TableId destinationTable = getDestinationTable(jobId); - return useReadAPI(totalRows, pageRows) + return useReadAPI(totalRows, pageRows, Schema.fromPb(firstPage.getSchema())) ? highThroughPutRead( destinationTable, firstPage.getTotalRows().longValue(), @@ -847,7 +849,14 @@ boolean isFastQuerySupported() { && connectionSettings.getWriteDisposition() == null; } - private boolean useReadAPI(Long totalRows, Long pageRows) { + private boolean useReadAPI(Long totalRows, Long pageRows, Schema schema) { + + if (!containsIntervalType( + schema)) { // finds out if there's an interval type in the schema. Implementation to be used + // until ReadAPI supports Interval + logger.log(Level.INFO, "\n Schema has IntervalType, Disabling ReadAPI"); + return false; + } long resultRatio = totalRows / pageRows; if (Boolean.TRUE.equals(connectionSettings.getUseReadAPI()) && connectionSettings.getReadClientConnectionConfiguration() @@ -863,6 +872,23 @@ private boolean useReadAPI(Long totalRows, Long pageRows) { } } + // Does a BFS iteration to find out if there's an interval type in the schema. Implementation to + // be used until ReadAPI supports IntervalType + private boolean containsIntervalType(Schema schema) { + Queue fields = + new LinkedList(schema.getFields()); + while (!fields.isEmpty()) { + com.google.cloud.bigquery.Field curField = fields.poll(); + if (curField.getType().getStandardType() == StandardSQLTypeName.INTERVAL) { + return true; + } else if (curField.getType().getStandardType() == StandardSQLTypeName.STRUCT + || curField.getType().getStandardType() == StandardSQLTypeName.ARRAY) { + fields.addAll(curField.getSubFields()); + } + } + return false; + } + // Used for job.query API endpoint private QueryRequest createQueryRequest( ConnectionSettings connectionSettings, From 84235765d55e8330430eddb7bc627850f601bce3 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 8 Mar 2022 15:25:08 +0530 Subject: [PATCH 154/277] Added containsIntervalType check --- .../main/java/com/google/cloud/bigquery/ConnectionImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 48c79eefc..478bbf30c 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -851,9 +851,9 @@ boolean isFastQuerySupported() { private boolean useReadAPI(Long totalRows, Long pageRows, Schema schema) { - if (!containsIntervalType( + if (containsIntervalType( schema)) { // finds out if there's an interval type in the schema. Implementation to be used - // until ReadAPI supports Interval + // until ReadAPI supports Interval logger.log(Level.INFO, "\n Schema has IntervalType, Disabling ReadAPI"); return false; } From d7f3de03f4e5a62bbc3daa87d56100c52dff2f03 Mon Sep 17 00:00:00 2001 From: stephwang Date: Tue, 8 Mar 2022 12:11:47 -0500 Subject: [PATCH 155/277] add back the arrow-netty dependency --- pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pom.xml b/pom.xml index 68f860cb2..b4715c8ae 100644 --- a/pom.xml +++ b/pom.xml @@ -196,6 +196,12 @@ arrow-memory-core 7.0.0 + + org.apache.arrow + arrow-memory-netty + 7.0.0 + runtime + From 9cb052785271b348a53094233022b99ee0228564 Mon Sep 17 00:00:00 2001 From: stephwang Date: Tue, 8 Mar 2022 12:19:01 -0500 Subject: [PATCH 156/277] add a rule to ignore org.apache.arrow:arrow-memory-netty in maven-dependency-plugin --- pom.xml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pom.xml b/pom.xml index b4715c8ae..6b90787fe 100644 --- a/pom.xml +++ b/pom.xml @@ -208,6 +208,21 @@ google-cloud-bigquery + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + org.apache.arrow:arrow-memory-netty + + + + + + From ca2e27ce1d512788686cb374419b45b8a328d61e Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 9 Mar 2022 14:27:24 +0530 Subject: [PATCH 157/277] Fixed getTime --- .../cloud/bigquery/BigQueryResultSetImpl.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index ffa91c299..5ad037055 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -24,7 +24,9 @@ import java.sql.Time; import java.sql.Timestamp; import java.time.LocalTime; +import java.time.ZoneId; import java.util.Map; +import java.util.TimeZone; import java.util.concurrent.BlockingQueue; import org.apache.arrow.vector.util.JsonStringArrayList; import org.apache.arrow.vector.util.Text; @@ -429,10 +431,20 @@ public Time getTime(String fieldName) throws SQLException { throw new SQLException(String.format("Field %s not found", fieldName)); } Object timeStampObj = curTuple.x().get(fieldName); - return timeStampObj == null ? null : new Time(((Long) timeStampObj).intValue()); + return timeStampObj == null + ? null + : new Time( + ((Long) timeStampObj) + / 1000); // Time.toString() will return 12:11:35 in GMT as 17:41:35 in + // (GMT+5:30). This can be offset using getTimeZoneOffset } } + private int getTimeZoneOffset() { + TimeZone timeZone = TimeZone.getTimeZone(ZoneId.systemDefault()); + return timeZone.getOffset(new java.util.Date().getTime()); // offset in seconds + } + @Override public Time getTime(int columnIndex) throws SQLException { if (cursor == null) { From 3eb7bc4e58b2e71f87bc496f81dec3f7856c3f7f Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 9 Mar 2022 14:44:27 +0530 Subject: [PATCH 158/277] Updated getString --- .../google/cloud/bigquery/BigQueryResultSetImpl.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index 5ad037055..8a6259b1c 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -137,7 +137,15 @@ public String getString(String fieldName) throws SQLException { return null; } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); - return fieldValue.getValue() == null ? null : fieldValue.getStringValue(); + if (fieldValue.getValue() == null) { + return null; + } else if (fieldValue + .getAttribute() + .equals(FieldValue.Attribute.REPEATED)) { // Case for Arrays + return fieldValue.getValue().toString(); + } else { + return fieldValue.getStringValue(); + } } else { // Data received from Read API (Arrow) Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; if (!curTuple.x().containsKey(fieldName)) { @@ -436,7 +444,7 @@ public Time getTime(String fieldName) throws SQLException { : new Time( ((Long) timeStampObj) / 1000); // Time.toString() will return 12:11:35 in GMT as 17:41:35 in - // (GMT+5:30). This can be offset using getTimeZoneOffset + // (GMT+5:30). This can be offset using getTimeZoneOffset } } From 03cc9ea595d684277545a957c939574fa7da7484 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Thu, 10 Mar 2022 00:21:03 +0000 Subject: [PATCH 159/277] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot?= =?UTF-8?q?=20post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f80dced69..420476374 100644 --- a/README.md +++ b/README.md @@ -59,13 +59,13 @@ implementation 'com.google.cloud:google-cloud-bigquery' If you are using Gradle without BOM, add this to your dependencies ```Groovy -implementation 'com.google.cloud:google-cloud-bigquery:2.9.3' +implementation 'com.google.cloud:google-cloud-bigquery:2.9.4' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.9.3" +libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.9.4" ``` ## Authentication From bbe20d5b6f506912a63d93482c4eb01694c66e7c Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 15 Mar 2022 11:27:59 +0530 Subject: [PATCH 160/277] Added Nightly Integration test for testing Read API integration --- .../com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java new file mode 100644 index 000000000..d12ee9af7 --- /dev/null +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java @@ -0,0 +1,2 @@ +package com.google.cloud.bigquery.it;public class ITNightlyBigQueryTest { +} From e049c382a5e210917cb635d5753a9ed533711fa2 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 15 Mar 2022 11:33:26 +0530 Subject: [PATCH 161/277] Added Nightly Integration test for testing Read API integration --- .../bigquery/it/ITNightlyBigQueryTest.java | 346 +++++++++++++++++- 1 file changed, 345 insertions(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java index d12ee9af7..4c70487f4 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java @@ -1,2 +1,346 @@ -package com.google.cloud.bigquery.it;public class ITNightlyBigQueryTest { +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery.it; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.cloud.ServiceOptions; +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQueryError; +import com.google.cloud.bigquery.BigQueryException; +import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.BigQueryResultSet; +import com.google.cloud.bigquery.Connection; +import com.google.cloud.bigquery.ConnectionSettings; +import com.google.cloud.bigquery.Dataset; +import com.google.cloud.bigquery.DatasetId; +import com.google.cloud.bigquery.DatasetInfo; +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.InsertAllRequest; +import com.google.cloud.bigquery.InsertAllResponse; +import com.google.cloud.bigquery.QueryJobConfiguration; +import com.google.cloud.bigquery.ReadClientConnectionConfiguration; +import com.google.cloud.bigquery.Schema; +import com.google.cloud.bigquery.StandardSQLTypeName; +import com.google.cloud.bigquery.StandardTableDefinition; +import com.google.cloud.bigquery.Table; +import com.google.cloud.bigquery.TableDefinition; +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.TableInfo; +import com.google.cloud.bigquery.testing.RemoteBigQueryHelper; +import java.io.IOException; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +public class ITNightlyBigQueryTest { + private static final Logger logger = Logger.getLogger(ITNightlyBigQueryTest.class.getName()); + private static final String DATASET = RemoteBigQueryHelper.generateDatasetName(); + private static final String TABLE = "TEMP_RS_TEST_TABLE"; + // Script will populate NUM_BATCHES*REC_PER_BATCHES number of records (eg: 100*10000 = 1M) + private static final int NUM_BATCHES = 100; + private static final int REC_PER_BATCHES = 10000; + private static final int LIMIT_RECS = 900000; + private static int rowCnt = 0; + private static final String PROJECT_ID = ServiceOptions.getDefaultProjectId(); + private static BigQuery bigquery; + private static final String QUERY = + "select StringField, GeographyField, BooleanField, BigNumericField, IntegerField, NumericField, BytesField, TimestampField, " + + " TimeField, DateField, IntegerArrayField, RecordField.BooleanField, RecordField.StringField , JSONField" + + " from BQ_RS_ARROW_TESTING.Table1 order by IntegerField asc LIMIT " + + LIMIT_RECS; + + private static final Schema BQ_SCHEMA = + Schema.of( + Field.newBuilder("TimestampField", StandardSQLTypeName.TIMESTAMP) + .setMode(Field.Mode.NULLABLE) + .setDescription("TimestampDescription") + .build(), + Field.newBuilder("StringField", StandardSQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .setDescription("StringDescription") + .build(), + Field.newBuilder("IntegerArrayField", StandardSQLTypeName.NUMERIC) + .setMode(Field.Mode.REPEATED) + .setDescription("IntegerArrayDescription") + .build(), + Field.newBuilder("BooleanField", StandardSQLTypeName.BOOL) + .setMode(Field.Mode.NULLABLE) + .setDescription("BooleanDescription") + .build(), + Field.newBuilder("BytesField", StandardSQLTypeName.BYTES) + .setMode(Field.Mode.NULLABLE) + .setDescription("BytesDescription") + .build(), + Field.newBuilder( + "RecordField", + StandardSQLTypeName.STRUCT, + Field.newBuilder("StringField", StandardSQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .setDescription("StringDescription") + .build(), + Field.newBuilder("BooleanField", StandardSQLTypeName.BOOL) + .setMode(Field.Mode.NULLABLE) + .setDescription("BooleanDescription") + .build()) + .setMode(Field.Mode.NULLABLE) + .setDescription("RecordDescription") + .build(), + Field.newBuilder("IntegerField", StandardSQLTypeName.NUMERIC) + .setMode(Field.Mode.NULLABLE) + .setDescription("IntegerDescription") + .build(), + Field.newBuilder("GeographyField", StandardSQLTypeName.GEOGRAPHY) + .setMode(Field.Mode.NULLABLE) + .setDescription("GeographyDescription") + .build(), + Field.newBuilder("NumericField", StandardSQLTypeName.NUMERIC) + .setMode(Field.Mode.NULLABLE) + .setDescription("NumericDescription") + .build(), + Field.newBuilder("BigNumericField", StandardSQLTypeName.BIGNUMERIC) + .setMode(Field.Mode.NULLABLE) + .setDescription("BigNumericDescription") + .build(), + Field.newBuilder("TimeField", StandardSQLTypeName.TIME) + .setMode(Field.Mode.NULLABLE) + .setDescription("TimeDescription") + .build(), + Field.newBuilder("DateField", StandardSQLTypeName.DATE) + .setMode(Field.Mode.NULLABLE) + .setDescription("DateDescription") + .build(), + Field.newBuilder("DateTimeField", StandardSQLTypeName.DATETIME) + .setMode(Field.Mode.NULLABLE) + .setDescription("DateTimeDescription") + .build(), + Field.newBuilder("JSONField", StandardSQLTypeName.JSON) + .setMode(Field.Mode.NULLABLE) + .setDescription("JSONFieldDescription") + .build(), + Field.newBuilder("IntervalField", StandardSQLTypeName.INTERVAL) + .setMode(Field.Mode.NULLABLE) + .setDescription("IntervalFieldDescription") + .build()); + + @Rule public Timeout globalTimeout = Timeout.seconds(1800); // setting 30 mins as the timeout + + @BeforeClass + public static void beforeClass() throws InterruptedException, IOException { + RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create(); + bigquery = bigqueryHelper.getOptions().getService(); + createDataset(DATASET); + createTable(DATASET, TABLE, BQ_SCHEMA); + populateTestRecords(DATASET, TABLE); + } + + @AfterClass + public static void afterClass() throws ExecutionException, InterruptedException { + try { + if (bigquery != null) { + deleteTable(DATASET, TABLE); + RemoteBigQueryHelper.forceDelete(bigquery, DATASET); + } else { + fail("Error clearing the test dataset"); + } + } catch (BigQueryException e) { + fail("Error clearing the test dataset " + e); + } + } + + @Test + public void testIterateAndOrder() throws SQLException { + ReadClientConnectionConfiguration clientConnectionConfiguration = + ReadClientConnectionConfiguration.newBuilder() + .setTotalToPageRowCountRatio(10L) + .setMinResultSize(200000L) + .setBufferSize(10000L) + .build(); + ConnectionSettings connectionSettings = + ConnectionSettings.newBuilder() + .setDefaultDataset(DatasetId.of("BQ_RS_ARROW_TESTING")) + .setNumBufferedRows(10000L) // page size + .setPriority( + QueryJobConfiguration.Priority + .INTERACTIVE) // so that isFastQuerySupported returns false + .setReadClientConnectionConfiguration(clientConnectionConfiguration) + .setUseReadAPI(true) + .build(); + Connection connection = + BigQueryOptions.getDefaultInstance().getService().createConnection(connectionSettings); + + BigQueryResultSet bigQueryResultSet = connection.executeSelect(QUERY); + ResultSet rs = bigQueryResultSet.getResultSet(); + int cnt = 0; + + assertTrue(rs.next()); // read first record, which is null + ++cnt; + assertNull(rs.getString("StringField")); + assertNull(rs.getString("GeographyField")); + assertNull(rs.getObject("IntegerArrayField")); + assertFalse(rs.getBoolean("BooleanField")); + assertTrue(0.0d == rs.getDouble("BigNumericField")); + assertTrue(0 == rs.getInt("IntegerField")); + assertTrue(0L == rs.getLong("NumericField")); + assertNull(rs.getBytes("BytesField")); + assertNull(rs.getTimestamp("TimestampField")); + assertNull(rs.getTime("TimeField")); + assertNull(rs.getDate("DateField")); + assertNull(rs.getString("JSONField")); + assertFalse(rs.getBoolean("BooleanField_1")); + assertNull(rs.getString("StringField_1")); + int prevIntegerFieldVal = 0; + while (rs.next()) { + ++cnt; + + assertNotNull(rs.getString("StringField")); + assertNotNull(rs.getString("GeographyField")); + assertNotNull(rs.getObject("IntegerArrayField")); + assertTrue(rs.getBoolean("BooleanField")); + assertTrue(0.0d < rs.getDouble("BigNumericField")); + assertTrue(0 < rs.getInt("IntegerField")); + assertTrue(0L < rs.getLong("NumericField")); + assertNotNull(rs.getBytes("BytesField")); + assertNotNull(rs.getTimestamp("TimestampField")); + assertNotNull(rs.getTime("TimeField")); + assertNotNull(rs.getDate("DateField")); + assertNotNull(rs.getString("JSONField")); + assertFalse(rs.getBoolean("BooleanField_1")); + assertNotNull(rs.getString("StringField_1")); + + // check the order of the records + assertTrue(prevIntegerFieldVal < rs.getInt("IntegerField")); + prevIntegerFieldVal = rs.getInt("IntegerField"); + } + + assertEquals(LIMIT_RECS, cnt); + } + + private static void populateTestRecords(String datasetName, String tableName) { + TableId tableId = TableId.of(datasetName, tableName); + for (int batchCnt = 1; batchCnt <= NUM_BATCHES; batchCnt++) { + addBatchRecords(tableId); + } + } + + private static void addBatchRecords(TableId tableId) { + Map nullRow = new HashMap<>(); + try { + InsertAllRequest.Builder reqBuilder = InsertAllRequest.newBuilder(tableId); + if (rowCnt == 0) { + reqBuilder.addRow(nullRow); + } + for (int i = 0; i < REC_PER_BATCHES; i++) { + reqBuilder.addRow(getNextRow()); + } + InsertAllResponse response = bigquery.insertAll(reqBuilder.build()); + + if (response.hasErrors()) { + // If any of the insertions failed, this lets you inspect the errors + for (Map.Entry> entry : response.getInsertErrors().entrySet()) { + logger.log( + Level.WARNING, + "Exception while adding records {0}", + entry.getValue()); // gracefully log it, it's okay if a few batches fail during IT + } + } + } catch (BigQueryException e) { + logger.log( + Level.WARNING, + "Exception while adding records {0}", + e); // gracefully log it, it's okay if a few batches fail during IT + } + } + + private static void createTable(String datasetName, String tableName, Schema schema) { + try { + TableId tableId = TableId.of(datasetName, tableName); + TableDefinition tableDefinition = StandardTableDefinition.of(schema); + TableInfo tableInfo = TableInfo.newBuilder(tableId, tableDefinition).build(); + Table table = bigquery.create(tableInfo); + assertTrue(table.exists()); + } catch (BigQueryException e) { + fail("Table was not created. \n" + e); + } + } + + public static void deleteTable(String datasetName, String tableName) { + try { + assertTrue(bigquery.delete(TableId.of(datasetName, tableName))); + } catch (BigQueryException e) { + fail("Table was not deleted. \n" + e); + } + } + + public static void createDataset(String datasetName) { + try { + DatasetInfo datasetInfo = DatasetInfo.newBuilder(datasetName).build(); + Dataset newDataset = bigquery.create(datasetInfo); + assertNotNull(newDataset.getDatasetId().getDataset()); + } catch (BigQueryException e) { + fail("Dataset was not created. \n" + e); + } + } + + public static void deleteDataset(String datasetName) { + try { + DatasetInfo datasetInfo = DatasetInfo.newBuilder(datasetName).build(); + assertTrue(bigquery.delete(datasetInfo.getDatasetId())); + } catch (BigQueryException e) { + fail("Dataset was not deleted. \n" + e); + } + } + + private static Map getNextRow() { + rowCnt++; + Map row = new HashMap<>(); + Map structVal = new HashMap<>(); + structVal.put("StringField", "Str Val " + rowCnt); + structVal.put("BooleanField", false); + row.put("RecordField", structVal); // struct + row.put("TimestampField", "2022-01-01 01:01:01"); + row.put("StringField", "String Val " + rowCnt); + row.put("IntegerArrayField", new int[] {1, 2, 3}); + row.put("BooleanField", true); + row.put("BytesField", "DQ4KDQ=="); + row.put("IntegerField", 1 + rowCnt); + row.put("GeographyField", "POINT(1 2)"); + row.put("NumericField", 100 + rowCnt); + row.put("BigNumericField", 10000000L + rowCnt); + row.put("TimeField", "12:11:35"); + row.put("DateField", "2022-01-01"); + row.put("DateTimeField", "0001-01-01 00:00:00"); + row.put("JSONField", "{\"hello\":\"world\"}"); + row.put("IntervalField", "10000-0 3660000 87840000:0:0"); + return row; + } } From 44b9f32a46f1b7aca4eb9f9031483050efde9e2b Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 15 Mar 2022 11:33:26 +0530 Subject: [PATCH 162/277] Added Nightly Integration test for testing Read API integration --- .../bigquery/it/ITNightlyBigQueryTest.java | 346 +++++++++++++++++- 1 file changed, 345 insertions(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java index d12ee9af7..4c70487f4 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java @@ -1,2 +1,346 @@ -package com.google.cloud.bigquery.it;public class ITNightlyBigQueryTest { +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery.it; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.cloud.ServiceOptions; +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQueryError; +import com.google.cloud.bigquery.BigQueryException; +import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.BigQueryResultSet; +import com.google.cloud.bigquery.Connection; +import com.google.cloud.bigquery.ConnectionSettings; +import com.google.cloud.bigquery.Dataset; +import com.google.cloud.bigquery.DatasetId; +import com.google.cloud.bigquery.DatasetInfo; +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.InsertAllRequest; +import com.google.cloud.bigquery.InsertAllResponse; +import com.google.cloud.bigquery.QueryJobConfiguration; +import com.google.cloud.bigquery.ReadClientConnectionConfiguration; +import com.google.cloud.bigquery.Schema; +import com.google.cloud.bigquery.StandardSQLTypeName; +import com.google.cloud.bigquery.StandardTableDefinition; +import com.google.cloud.bigquery.Table; +import com.google.cloud.bigquery.TableDefinition; +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.TableInfo; +import com.google.cloud.bigquery.testing.RemoteBigQueryHelper; +import java.io.IOException; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +public class ITNightlyBigQueryTest { + private static final Logger logger = Logger.getLogger(ITNightlyBigQueryTest.class.getName()); + private static final String DATASET = RemoteBigQueryHelper.generateDatasetName(); + private static final String TABLE = "TEMP_RS_TEST_TABLE"; + // Script will populate NUM_BATCHES*REC_PER_BATCHES number of records (eg: 100*10000 = 1M) + private static final int NUM_BATCHES = 100; + private static final int REC_PER_BATCHES = 10000; + private static final int LIMIT_RECS = 900000; + private static int rowCnt = 0; + private static final String PROJECT_ID = ServiceOptions.getDefaultProjectId(); + private static BigQuery bigquery; + private static final String QUERY = + "select StringField, GeographyField, BooleanField, BigNumericField, IntegerField, NumericField, BytesField, TimestampField, " + + " TimeField, DateField, IntegerArrayField, RecordField.BooleanField, RecordField.StringField , JSONField" + + " from BQ_RS_ARROW_TESTING.Table1 order by IntegerField asc LIMIT " + + LIMIT_RECS; + + private static final Schema BQ_SCHEMA = + Schema.of( + Field.newBuilder("TimestampField", StandardSQLTypeName.TIMESTAMP) + .setMode(Field.Mode.NULLABLE) + .setDescription("TimestampDescription") + .build(), + Field.newBuilder("StringField", StandardSQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .setDescription("StringDescription") + .build(), + Field.newBuilder("IntegerArrayField", StandardSQLTypeName.NUMERIC) + .setMode(Field.Mode.REPEATED) + .setDescription("IntegerArrayDescription") + .build(), + Field.newBuilder("BooleanField", StandardSQLTypeName.BOOL) + .setMode(Field.Mode.NULLABLE) + .setDescription("BooleanDescription") + .build(), + Field.newBuilder("BytesField", StandardSQLTypeName.BYTES) + .setMode(Field.Mode.NULLABLE) + .setDescription("BytesDescription") + .build(), + Field.newBuilder( + "RecordField", + StandardSQLTypeName.STRUCT, + Field.newBuilder("StringField", StandardSQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .setDescription("StringDescription") + .build(), + Field.newBuilder("BooleanField", StandardSQLTypeName.BOOL) + .setMode(Field.Mode.NULLABLE) + .setDescription("BooleanDescription") + .build()) + .setMode(Field.Mode.NULLABLE) + .setDescription("RecordDescription") + .build(), + Field.newBuilder("IntegerField", StandardSQLTypeName.NUMERIC) + .setMode(Field.Mode.NULLABLE) + .setDescription("IntegerDescription") + .build(), + Field.newBuilder("GeographyField", StandardSQLTypeName.GEOGRAPHY) + .setMode(Field.Mode.NULLABLE) + .setDescription("GeographyDescription") + .build(), + Field.newBuilder("NumericField", StandardSQLTypeName.NUMERIC) + .setMode(Field.Mode.NULLABLE) + .setDescription("NumericDescription") + .build(), + Field.newBuilder("BigNumericField", StandardSQLTypeName.BIGNUMERIC) + .setMode(Field.Mode.NULLABLE) + .setDescription("BigNumericDescription") + .build(), + Field.newBuilder("TimeField", StandardSQLTypeName.TIME) + .setMode(Field.Mode.NULLABLE) + .setDescription("TimeDescription") + .build(), + Field.newBuilder("DateField", StandardSQLTypeName.DATE) + .setMode(Field.Mode.NULLABLE) + .setDescription("DateDescription") + .build(), + Field.newBuilder("DateTimeField", StandardSQLTypeName.DATETIME) + .setMode(Field.Mode.NULLABLE) + .setDescription("DateTimeDescription") + .build(), + Field.newBuilder("JSONField", StandardSQLTypeName.JSON) + .setMode(Field.Mode.NULLABLE) + .setDescription("JSONFieldDescription") + .build(), + Field.newBuilder("IntervalField", StandardSQLTypeName.INTERVAL) + .setMode(Field.Mode.NULLABLE) + .setDescription("IntervalFieldDescription") + .build()); + + @Rule public Timeout globalTimeout = Timeout.seconds(1800); // setting 30 mins as the timeout + + @BeforeClass + public static void beforeClass() throws InterruptedException, IOException { + RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create(); + bigquery = bigqueryHelper.getOptions().getService(); + createDataset(DATASET); + createTable(DATASET, TABLE, BQ_SCHEMA); + populateTestRecords(DATASET, TABLE); + } + + @AfterClass + public static void afterClass() throws ExecutionException, InterruptedException { + try { + if (bigquery != null) { + deleteTable(DATASET, TABLE); + RemoteBigQueryHelper.forceDelete(bigquery, DATASET); + } else { + fail("Error clearing the test dataset"); + } + } catch (BigQueryException e) { + fail("Error clearing the test dataset " + e); + } + } + + @Test + public void testIterateAndOrder() throws SQLException { + ReadClientConnectionConfiguration clientConnectionConfiguration = + ReadClientConnectionConfiguration.newBuilder() + .setTotalToPageRowCountRatio(10L) + .setMinResultSize(200000L) + .setBufferSize(10000L) + .build(); + ConnectionSettings connectionSettings = + ConnectionSettings.newBuilder() + .setDefaultDataset(DatasetId.of("BQ_RS_ARROW_TESTING")) + .setNumBufferedRows(10000L) // page size + .setPriority( + QueryJobConfiguration.Priority + .INTERACTIVE) // so that isFastQuerySupported returns false + .setReadClientConnectionConfiguration(clientConnectionConfiguration) + .setUseReadAPI(true) + .build(); + Connection connection = + BigQueryOptions.getDefaultInstance().getService().createConnection(connectionSettings); + + BigQueryResultSet bigQueryResultSet = connection.executeSelect(QUERY); + ResultSet rs = bigQueryResultSet.getResultSet(); + int cnt = 0; + + assertTrue(rs.next()); // read first record, which is null + ++cnt; + assertNull(rs.getString("StringField")); + assertNull(rs.getString("GeographyField")); + assertNull(rs.getObject("IntegerArrayField")); + assertFalse(rs.getBoolean("BooleanField")); + assertTrue(0.0d == rs.getDouble("BigNumericField")); + assertTrue(0 == rs.getInt("IntegerField")); + assertTrue(0L == rs.getLong("NumericField")); + assertNull(rs.getBytes("BytesField")); + assertNull(rs.getTimestamp("TimestampField")); + assertNull(rs.getTime("TimeField")); + assertNull(rs.getDate("DateField")); + assertNull(rs.getString("JSONField")); + assertFalse(rs.getBoolean("BooleanField_1")); + assertNull(rs.getString("StringField_1")); + int prevIntegerFieldVal = 0; + while (rs.next()) { + ++cnt; + + assertNotNull(rs.getString("StringField")); + assertNotNull(rs.getString("GeographyField")); + assertNotNull(rs.getObject("IntegerArrayField")); + assertTrue(rs.getBoolean("BooleanField")); + assertTrue(0.0d < rs.getDouble("BigNumericField")); + assertTrue(0 < rs.getInt("IntegerField")); + assertTrue(0L < rs.getLong("NumericField")); + assertNotNull(rs.getBytes("BytesField")); + assertNotNull(rs.getTimestamp("TimestampField")); + assertNotNull(rs.getTime("TimeField")); + assertNotNull(rs.getDate("DateField")); + assertNotNull(rs.getString("JSONField")); + assertFalse(rs.getBoolean("BooleanField_1")); + assertNotNull(rs.getString("StringField_1")); + + // check the order of the records + assertTrue(prevIntegerFieldVal < rs.getInt("IntegerField")); + prevIntegerFieldVal = rs.getInt("IntegerField"); + } + + assertEquals(LIMIT_RECS, cnt); + } + + private static void populateTestRecords(String datasetName, String tableName) { + TableId tableId = TableId.of(datasetName, tableName); + for (int batchCnt = 1; batchCnt <= NUM_BATCHES; batchCnt++) { + addBatchRecords(tableId); + } + } + + private static void addBatchRecords(TableId tableId) { + Map nullRow = new HashMap<>(); + try { + InsertAllRequest.Builder reqBuilder = InsertAllRequest.newBuilder(tableId); + if (rowCnt == 0) { + reqBuilder.addRow(nullRow); + } + for (int i = 0; i < REC_PER_BATCHES; i++) { + reqBuilder.addRow(getNextRow()); + } + InsertAllResponse response = bigquery.insertAll(reqBuilder.build()); + + if (response.hasErrors()) { + // If any of the insertions failed, this lets you inspect the errors + for (Map.Entry> entry : response.getInsertErrors().entrySet()) { + logger.log( + Level.WARNING, + "Exception while adding records {0}", + entry.getValue()); // gracefully log it, it's okay if a few batches fail during IT + } + } + } catch (BigQueryException e) { + logger.log( + Level.WARNING, + "Exception while adding records {0}", + e); // gracefully log it, it's okay if a few batches fail during IT + } + } + + private static void createTable(String datasetName, String tableName, Schema schema) { + try { + TableId tableId = TableId.of(datasetName, tableName); + TableDefinition tableDefinition = StandardTableDefinition.of(schema); + TableInfo tableInfo = TableInfo.newBuilder(tableId, tableDefinition).build(); + Table table = bigquery.create(tableInfo); + assertTrue(table.exists()); + } catch (BigQueryException e) { + fail("Table was not created. \n" + e); + } + } + + public static void deleteTable(String datasetName, String tableName) { + try { + assertTrue(bigquery.delete(TableId.of(datasetName, tableName))); + } catch (BigQueryException e) { + fail("Table was not deleted. \n" + e); + } + } + + public static void createDataset(String datasetName) { + try { + DatasetInfo datasetInfo = DatasetInfo.newBuilder(datasetName).build(); + Dataset newDataset = bigquery.create(datasetInfo); + assertNotNull(newDataset.getDatasetId().getDataset()); + } catch (BigQueryException e) { + fail("Dataset was not created. \n" + e); + } + } + + public static void deleteDataset(String datasetName) { + try { + DatasetInfo datasetInfo = DatasetInfo.newBuilder(datasetName).build(); + assertTrue(bigquery.delete(datasetInfo.getDatasetId())); + } catch (BigQueryException e) { + fail("Dataset was not deleted. \n" + e); + } + } + + private static Map getNextRow() { + rowCnt++; + Map row = new HashMap<>(); + Map structVal = new HashMap<>(); + structVal.put("StringField", "Str Val " + rowCnt); + structVal.put("BooleanField", false); + row.put("RecordField", structVal); // struct + row.put("TimestampField", "2022-01-01 01:01:01"); + row.put("StringField", "String Val " + rowCnt); + row.put("IntegerArrayField", new int[] {1, 2, 3}); + row.put("BooleanField", true); + row.put("BytesField", "DQ4KDQ=="); + row.put("IntegerField", 1 + rowCnt); + row.put("GeographyField", "POINT(1 2)"); + row.put("NumericField", 100 + rowCnt); + row.put("BigNumericField", 10000000L + rowCnt); + row.put("TimeField", "12:11:35"); + row.put("DateField", "2022-01-01"); + row.put("DateTimeField", "0001-01-01 00:00:00"); + row.put("JSONField", "{\"hello\":\"world\"}"); + row.put("IntervalField", "10000-0 3660000 87840000:0:0"); + return row; + } } From 708fa533631e4e86c54b0e33bf886a43893ea65d Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 15 Mar 2022 19:42:27 +0530 Subject: [PATCH 163/277] Modified Nightly Integration test --- .../bigquery/it/ITNightlyBigQueryTest.java | 95 ++++++++++--------- 1 file changed, 48 insertions(+), 47 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java index 4c70487f4..a1ee848b7 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java @@ -23,11 +23,9 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import com.google.cloud.ServiceOptions; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryError; import com.google.cloud.bigquery.BigQueryException; -import com.google.cloud.bigquery.BigQueryOptions; import com.google.cloud.bigquery.BigQueryResultSet; import com.google.cloud.bigquery.Connection; import com.google.cloud.bigquery.ConnectionSettings; @@ -56,6 +54,7 @@ import java.util.concurrent.ExecutionException; import java.util.logging.Level; import java.util.logging.Logger; +import org.apache.arrow.vector.util.JsonStringArrayList; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Rule; @@ -67,17 +66,16 @@ public class ITNightlyBigQueryTest { private static final String DATASET = RemoteBigQueryHelper.generateDatasetName(); private static final String TABLE = "TEMP_RS_TEST_TABLE"; // Script will populate NUM_BATCHES*REC_PER_BATCHES number of records (eg: 100*10000 = 1M) - private static final int NUM_BATCHES = 100; + private static final int NUM_BATCHES = 55; private static final int REC_PER_BATCHES = 10000; - private static final int LIMIT_RECS = 900000; + private static final int LIMIT_RECS = 500000; private static int rowCnt = 0; - private static final String PROJECT_ID = ServiceOptions.getDefaultProjectId(); private static BigQuery bigquery; private static final String QUERY = - "select StringField, GeographyField, BooleanField, BigNumericField, IntegerField, NumericField, BytesField, TimestampField, " - + " TimeField, DateField, IntegerArrayField, RecordField.BooleanField, RecordField.StringField , JSONField" - + " from BQ_RS_ARROW_TESTING.Table1 order by IntegerField asc LIMIT " - + LIMIT_RECS; + String.format( + "select StringField, GeographyField, BooleanField, BigNumericField, IntegerField, NumericField, BytesField, TimestampField, TimeField, DateField, " + + "IntegerArrayField, RecordField.BooleanField, RecordField.StringField , JSONField from %s order by IntegerField asc LIMIT %s", + TABLE, LIMIT_RECS); private static final Schema BQ_SCHEMA = Schema.of( @@ -187,7 +185,7 @@ public void testIterateAndOrder() throws SQLException { .build(); ConnectionSettings connectionSettings = ConnectionSettings.newBuilder() - .setDefaultDataset(DatasetId.of("BQ_RS_ARROW_TESTING")) + .setDefaultDataset(DatasetId.of(DATASET)) .setNumBufferedRows(10000L) // page size .setPriority( QueryJobConfiguration.Priority @@ -195,53 +193,56 @@ public void testIterateAndOrder() throws SQLException { .setReadClientConnectionConfiguration(clientConnectionConfiguration) .setUseReadAPI(true) .build(); - Connection connection = - BigQueryOptions.getDefaultInstance().getService().createConnection(connectionSettings); + Connection connection = bigquery.createConnection(connectionSettings); BigQueryResultSet bigQueryResultSet = connection.executeSelect(QUERY); ResultSet rs = bigQueryResultSet.getResultSet(); int cnt = 0; - assertTrue(rs.next()); // read first record, which is null - ++cnt; - assertNull(rs.getString("StringField")); - assertNull(rs.getString("GeographyField")); - assertNull(rs.getObject("IntegerArrayField")); - assertFalse(rs.getBoolean("BooleanField")); - assertTrue(0.0d == rs.getDouble("BigNumericField")); - assertTrue(0 == rs.getInt("IntegerField")); - assertTrue(0L == rs.getLong("NumericField")); - assertNull(rs.getBytes("BytesField")); - assertNull(rs.getTimestamp("TimestampField")); - assertNull(rs.getTime("TimeField")); - assertNull(rs.getDate("DateField")); - assertNull(rs.getString("JSONField")); - assertFalse(rs.getBoolean("BooleanField_1")); - assertNull(rs.getString("StringField_1")); int prevIntegerFieldVal = 0; while (rs.next()) { ++cnt; + if (cnt == 1) { // first row is supposed to be null + assertNull(rs.getString("StringField")); + assertNull(rs.getString("GeographyField")); + Object intAryField = rs.getObject("IntegerArrayField"); + if (intAryField instanceof JsonStringArrayList) { + assertEquals( + new JsonStringArrayList(), + ((JsonStringArrayList) intAryField)); // null array is returned as an empty array + } + assertFalse(rs.getBoolean("BooleanField")); + assertTrue(0.0d == rs.getDouble("BigNumericField")); + assertTrue(0 == rs.getInt("IntegerField")); + assertTrue(0L == rs.getLong("NumericField")); + assertNull(rs.getBytes("BytesField")); + assertNull(rs.getTimestamp("TimestampField")); + assertNull(rs.getTime("TimeField")); + assertNull(rs.getDate("DateField")); + assertNull(rs.getString("JSONField")); + assertFalse(rs.getBoolean("BooleanField_1")); + assertNull(rs.getString("StringField_1")); + } else { // remaining rows are supposed to be non null + assertNotNull(rs.getString("StringField")); + assertNotNull(rs.getString("GeographyField")); + assertNotNull(rs.getObject("IntegerArrayField")); + assertTrue(rs.getBoolean("BooleanField")); + assertTrue(0.0d < rs.getDouble("BigNumericField")); + assertTrue(0 < rs.getInt("IntegerField")); + assertTrue(0L < rs.getLong("NumericField")); + assertNotNull(rs.getBytes("BytesField")); + assertNotNull(rs.getTimestamp("TimestampField")); + assertNotNull(rs.getTime("TimeField")); + assertNotNull(rs.getDate("DateField")); + assertNotNull(rs.getString("JSONField")); + assertFalse(rs.getBoolean("BooleanField_1")); + assertNotNull(rs.getString("StringField_1")); - assertNotNull(rs.getString("StringField")); - assertNotNull(rs.getString("GeographyField")); - assertNotNull(rs.getObject("IntegerArrayField")); - assertTrue(rs.getBoolean("BooleanField")); - assertTrue(0.0d < rs.getDouble("BigNumericField")); - assertTrue(0 < rs.getInt("IntegerField")); - assertTrue(0L < rs.getLong("NumericField")); - assertNotNull(rs.getBytes("BytesField")); - assertNotNull(rs.getTimestamp("TimestampField")); - assertNotNull(rs.getTime("TimeField")); - assertNotNull(rs.getDate("DateField")); - assertNotNull(rs.getString("JSONField")); - assertFalse(rs.getBoolean("BooleanField_1")); - assertNotNull(rs.getString("StringField_1")); - - // check the order of the records - assertTrue(prevIntegerFieldVal < rs.getInt("IntegerField")); - prevIntegerFieldVal = rs.getInt("IntegerField"); + // check the order of the records + assertTrue(prevIntegerFieldVal < rs.getInt("IntegerField")); + prevIntegerFieldVal = rs.getInt("IntegerField"); + } } - assertEquals(LIMIT_RECS, cnt); } From 90ce59457b3c1adc13a09e68ccd0e4ff7cdc1f46 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 15 Mar 2022 19:44:52 +0530 Subject: [PATCH 164/277] Fixed NPE during bulk read --- .../google/cloud/bigquery/ConnectionImpl.java | 69 ++++++++++++++++--- 1 file changed, 58 insertions(+), 11 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 478bbf30c..567a4b275 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -25,7 +25,6 @@ import com.google.api.services.bigquery.model.QueryRequest; import com.google.api.services.bigquery.model.TableDataList; import com.google.api.services.bigquery.model.TableRow; -import com.google.api.services.bigquery.model.TableSchema; import com.google.cloud.RetryHelper; import com.google.cloud.Tuple; import com.google.cloud.bigquery.spi.v2.BigQueryRpc; @@ -151,8 +150,26 @@ public BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException { createQueryJob(sql, connectionSettings, null, null); JobId jobId = JobId.fromPb(queryJob.getJobReference()); GetQueryResultsResponse firstPage = getQueryResultsFirstPage(jobId); - return getSubsequentQueryResultsWithJob( - firstPage.getTotalRows().longValue(), (long) firstPage.getRows().size(), jobId, firstPage); + return getResultSet(firstPage, jobId, sql); + } + + private BigQueryResultSet getResultSet( + GetQueryResultsResponse firstPage, JobId jobId, String sql) { + if (firstPage.getJobComplete() + && firstPage.getTotalRows() + != null) { // firstPage.getTotalRows() is null if job is not complete + return getSubsequentQueryResultsWithJob( + firstPage.getTotalRows().longValue(), + (long) firstPage.getRows().size(), + jobId, + firstPage); + } else { // job is still running, use dryrun to get Schema + com.google.api.services.bigquery.model.Job dryRunJob = createDryRunJob(sql); + Schema schema = Schema.fromPb(dryRunJob.getStatistics().getQuery().getSchema()); + // TODO: check how can we get totalRows and pageRows while the job is still running. + // `firstPage.getTotalRows()` returns null + return getSubsequentQueryResultsWithJob(null, null, jobId, firstPage, schema); + } } /** @@ -185,8 +202,7 @@ public BigQueryResultSet executeSelect( createQueryJob(sql, connectionSettings, parameters, labels); JobId jobId = JobId.fromPb(queryJob.getJobReference()); GetQueryResultsResponse firstPage = getQueryResultsFirstPage(jobId); - return getSubsequentQueryResultsWithJob( - firstPage.getTotalRows().longValue(), (long) firstPage.getRows().size(), jobId, firstPage); + return getResultSet(firstPage, jobId, sql); } static class EndOfFieldValueList @@ -538,7 +554,30 @@ BigQueryResultSet getSubsequentQueryResultsWithJob( ? highThroughPutRead( destinationTable, firstPage.getTotalRows().longValue(), - firstPage.getSchema(), + Schema.fromPb(firstPage.getSchema()), + getBigQueryResultSetStats( + jobId)) // discord first page and stream the entire BigQueryResultSet using + // the Read API + : tableDataList(firstPage, jobId); + } + + /* Returns query results using either tabledata.list or the high throughput Read API */ + private BigQueryResultSet getSubsequentQueryResultsWithJob( + Long totalRows, + Long pageRows, + JobId jobId, + GetQueryResultsResponse firstPage, + Schema schema) { + TableId destinationTable = getDestinationTable(jobId); + return useReadAPI(totalRows, pageRows, schema) + ? highThroughPutRead( + destinationTable, + totalRows == null + ? -1L + : totalRows + .longValue(), // totalRows is null when the job is still running. TODO: Check if + // any workaround is possible + schema, getBigQueryResultSetStats( jobId)) // discord first page and stream the entire BigQueryResultSet using // the Read API @@ -612,10 +651,7 @@ TableDataList tableDataListRpc(TableId destinationTable, String pageToken) { } private BigQueryResultSet highThroughPutRead( - TableId destinationTable, - long totalRows, - TableSchema tableSchema, - BigQueryResultSetStats stats) { + TableId destinationTable, long totalRows, Schema schema, BigQueryResultSetStats stats) { try { if (bqReadClient == null) { // if the read client isn't already initialized. Not thread safe. @@ -646,7 +682,6 @@ private BigQueryResultSet highThroughPutRead( BlockingQueue, Boolean>> buffer = new LinkedBlockingDeque<>(bufferSize); Map arrowNameToIndex = new HashMap<>(); - Schema schema = Schema.fromPb(tableSchema); // deserialize and populate the buffer async, so that the client isn't blocked processArrowStreamAsync( readSession, @@ -851,12 +886,24 @@ boolean isFastQuerySupported() { private boolean useReadAPI(Long totalRows, Long pageRows, Schema schema) { + // TODO(prasmish) get this logic review - totalRows and pageRows are returned null when the job + // is not complete + if ((totalRows == null || pageRows == null) + && Boolean.TRUE.equals( + connectionSettings + .getUseReadAPI())) { // totalRows and pageRows are returned null when the job is not + // complete + return true; + } + + // Schema schema = Schema.fromPb(tableSchema); if (containsIntervalType( schema)) { // finds out if there's an interval type in the schema. Implementation to be used // until ReadAPI supports Interval logger.log(Level.INFO, "\n Schema has IntervalType, Disabling ReadAPI"); return false; } + long resultRatio = totalRows / pageRows; if (Boolean.TRUE.equals(connectionSettings.getUseReadAPI()) && connectionSettings.getReadClientConnectionConfiguration() From f427725abcf9534e3080eb76af5fd88e623e1cb8 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 15 Mar 2022 20:29:36 +0530 Subject: [PATCH 165/277] Fixed class cast exception @ getLong --- .../com/google/cloud/bigquery/BigQueryResultSetImpl.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index 8a6259b1c..ff8b1c780 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -229,7 +229,13 @@ public long getLong(String fieldName) throws SQLException { throw new SQLException(String.format("Field %s not found", fieldName)); } Object curVal = curTuple.x().get(fieldName); - return curVal == null ? 0 : ((BigDecimal) curVal).longValue(); + if (curVal == null) { + return 0L; + } else if (curVal instanceof Long) { + return ((Long) curVal).longValue(); + } else { + return ((BigDecimal) curVal).longValue(); + } } } From 4c7bfc75f313645877c8fd5e47e628974e24bc0f Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 16 Mar 2022 10:49:37 +0530 Subject: [PATCH 166/277] Added @BetaApi Annotation --- .../src/main/java/com/google/cloud/bigquery/Connection.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java index f254cadc0..18b32f363 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java @@ -16,6 +16,7 @@ package com.google.cloud.bigquery; +import com.google.api.core.BetaApi; import com.google.api.services.bigquery.model.QueryParameter; import java.util.List; import java.util.Map; @@ -27,6 +28,7 @@ public interface Connection { /** Sends a query cancel request. This call will return immediately */ + @BetaApi Boolean cancel() throws BigQuerySQLException; /** @@ -36,6 +38,7 @@ public interface Connection { * @param sql typically a static SQL SELECT statement * @exception BigQuerySQLException if a database access error occurs */ + @BetaApi BigQueryDryRunResult dryRun(String sql) throws BigQuerySQLException; /** @@ -69,6 +72,7 @@ public interface Connection { * @return a ResultSet that contains the data produced by the query * @exception BigQuerySQLException if a database access error occurs */ + @BetaApi BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException; /** @@ -86,6 +90,7 @@ public interface Connection { * @return a ResultSet that contains the data produced by the query * @exception BigQuerySQLException if a database access error occurs */ + @BetaApi BigQueryResultSet executeSelect( String sql, List parameters, Map labels) throws BigQuerySQLException; From 82f67459b41bfbf055214d8be51f4dc7c38311a2 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 16 Mar 2022 10:54:38 +0530 Subject: [PATCH 167/277] Added @BetaApi Annotation --- .../java/com/google/cloud/bigquery/ConnectionImpl.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 567a4b275..e81150fef 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -19,6 +19,7 @@ import static com.google.cloud.RetryHelper.runWithRetries; import static java.net.HttpURLConnection.HTTP_NOT_FOUND; +import com.google.api.core.BetaApi; import com.google.api.services.bigquery.model.GetQueryResultsResponse; import com.google.api.services.bigquery.model.JobConfigurationQuery; import com.google.api.services.bigquery.model.QueryParameter; @@ -108,6 +109,7 @@ class ConnectionImpl implements Connection { * @return Boolean value true if the threads were interrupted * @throws BigQuerySQLException */ + @BetaApi @Override public synchronized Boolean cancel() throws BigQuerySQLException { queryTaskExecutor.shutdownNow(); @@ -121,6 +123,7 @@ public synchronized Boolean cancel() throws BigQuerySQLException { * @return BigQueryDryRunResult containing List and Schema * @throws BigQuerySQLException */ + @BetaApi @Override public BigQueryDryRunResult dryRun(String sql) throws BigQuerySQLException { com.google.api.services.bigquery.model.Job dryRunJob = createDryRunJob(sql); @@ -137,6 +140,7 @@ public BigQueryDryRunResult dryRun(String sql) throws BigQuerySQLException { * @return BigQueryResultSet containing the output of the query * @throws BigQuerySQLException */ + @BetaApi @Override public BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException { // use jobs.query if all the properties of connectionSettings are supported @@ -186,6 +190,7 @@ private BigQueryResultSet getResultSet( * @return BigQueryResultSet containing the output of the query * @throws BigQuerySQLException */ + @BetaApi @Override public BigQueryResultSet executeSelect( String sql, List parameters, Map labels) @@ -747,7 +752,7 @@ private class ArrowRowReader private final VectorSchemaRoot root; private final VectorLoader loader; - public ArrowRowReader(ArrowSchema arrowSchema, Map arrowNameToIndex) + private ArrowRowReader(ArrowSchema arrowSchema, Map arrowNameToIndex) throws IOException { org.apache.arrow.vector.types.pojo.Schema schema = MessageSerializer.deserializeSchema( @@ -767,7 +772,7 @@ public ArrowRowReader(ArrowSchema arrowSchema, Map arrowNameToI } /** @param batch object returned from the ReadRowsResponse. */ - public void processRows( + private void processRows( ArrowRecordBatch batch, BlockingQueue, Boolean>> buffer, Schema schema) From 78d691c103df857a3c6f75e2d6873ed81926dbff Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 17 Mar 2022 13:18:31 +0530 Subject: [PATCH 168/277] Exposed methods for unit testing --- .../google/cloud/bigquery/ConnectionImpl.java | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index e81150fef..9d20eeecd 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -157,8 +157,8 @@ public BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException { return getResultSet(firstPage, jobId, sql); } - private BigQueryResultSet getResultSet( - GetQueryResultsResponse firstPage, JobId jobId, String sql) { + @VisibleForTesting + BigQueryResultSet getResultSet(GetQueryResultsResponse firstPage, JobId jobId, String sql) { if (firstPage.getJobComplete() && firstPage.getTotalRows() != null) { // firstPage.getTotalRows() is null if job is not complete @@ -264,7 +264,8 @@ private BigQueryResultSet queryRpc(final String projectId, final QueryRequest qu } } - private BigQueryResultSetStats getBigQueryResultSetStats(JobId jobId) { + @VisibleForTesting + BigQueryResultSetStats getBigQueryResultSetStats(JobId jobId) { // Create GetQueryResultsResponse query statistics Job queryJob = getQueryJobRpc(jobId); JobStatistics.QueryStatistics statistics = queryJob.getStatistics(); @@ -521,7 +522,8 @@ public FieldValueList apply(TableRow rowPb) { } /* Helper method that determines the optimal number of caches pages to improve read performance */ - private int getPageCacheSize(Long numBufferedRows, long numRows, Schema schema) { + @VisibleForTesting + int getPageCacheSize(Long numBufferedRows, long numRows, Schema schema) { final int MIN_CACHE_SIZE = 3; // Min number of pages in the page size final int MAX_CACHE_SIZE = 20; // //Min number of pages in the page size int columnsRead = schema.getFields().size(); @@ -567,7 +569,8 @@ BigQueryResultSet getSubsequentQueryResultsWithJob( } /* Returns query results using either tabledata.list or the high throughput Read API */ - private BigQueryResultSet getSubsequentQueryResultsWithJob( + @VisibleForTesting + BigQueryResultSet getSubsequentQueryResultsWithJob( Long totalRows, Long pageRows, JobId jobId, @@ -655,7 +658,8 @@ TableDataList tableDataListRpc(TableId destinationTable, String pageToken) { } } - private BigQueryResultSet highThroughPutRead( + @VisibleForTesting + BigQueryResultSet highThroughPutRead( TableId destinationTable, long totalRows, Schema schema, BigQueryResultSetStats stats) { try { @@ -889,7 +893,8 @@ boolean isFastQuerySupported() { && connectionSettings.getWriteDisposition() == null; } - private boolean useReadAPI(Long totalRows, Long pageRows, Schema schema) { + @VisibleForTesting + boolean useReadAPI(Long totalRows, Long pageRows, Schema schema) { // TODO(prasmish) get this logic review - totalRows and pageRows are returned null when the job // is not complete @@ -942,7 +947,8 @@ private boolean containsIntervalType(Schema schema) { } // Used for job.query API endpoint - private QueryRequest createQueryRequest( + @VisibleForTesting + QueryRequest createQueryRequest( ConnectionSettings connectionSettings, String sql, List queryParameters, @@ -982,7 +988,8 @@ private QueryRequest createQueryRequest( } // Used by jobs.getQueryResults API endpoint - private com.google.api.services.bigquery.model.Job createQueryJob( + @VisibleForTesting + com.google.api.services.bigquery.model.Job createQueryJob( String sql, ConnectionSettings connectionSettings, List queryParameters, From f24e4c246aa15e39b7124b450db90e31459ae991 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 17 Mar 2022 13:19:05 +0530 Subject: [PATCH 169/277] Added more unit testing --- .../cloud/bigquery/ConnectionImplTest.java | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java index 3beff4ebc..81381ed86 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java @@ -17,6 +17,7 @@ package com.google.cloud.bigquery; import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.*; import static org.mockito.Mockito.verify; @@ -31,7 +32,10 @@ import java.math.BigInteger; import java.sql.SQLException; import java.util.AbstractList; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingDeque; import org.junit.Before; @@ -416,4 +420,101 @@ public void testLegacyQueryMultiplePages() throws SQLException { verify(connectionSpy, times(1)) .tableDataList(any(GetQueryResultsResponse.class), any(JobId.class)); } + + @Test + public void testExecuteSelectSlow() throws BigQuerySQLException { + ConnectionImpl connectionSpy = Mockito.spy(connection); + doReturn(false).when(connectionSpy).isFastQuerySupported(); + com.google.api.services.bigquery.model.JobStatistics jobStatistics = + new com.google.api.services.bigquery.model.JobStatistics(); + com.google.api.services.bigquery.model.Job jobResponseMock = + new com.google.api.services.bigquery.model.Job() + .setJobReference(QUERY_JOB.toPb()) + .setId(JOB) + .setStatus(new com.google.api.services.bigquery.model.JobStatus().setState("DONE")) + .setStatistics(jobStatistics); + + doReturn(jobResponseMock) + .when(connectionSpy) + .createQueryJob(SQL_QUERY, connectionSettings, null, null); + doReturn(GET_QUERY_RESULTS_RESPONSE) + .when(connectionSpy) + .getQueryResultsFirstPage(any(JobId.class)); + doReturn(BQ_RS_MOCK_RES) + .when(connectionSpy) + .getResultSet(any(GetQueryResultsResponse.class), any(JobId.class), any(String.class)); + BigQueryResultSet res = connectionSpy.executeSelect(SQL_QUERY); + assertEquals(res.getTotalRows(), 2); + assertEquals(QUERY_SCHEMA, res.getSchema()); + verify(connectionSpy, times(1)) + .getResultSet(any(GetQueryResultsResponse.class), any(JobId.class), any(String.class)); + } + + @Test + public void testExecuteSelectSlowWithParams() throws BigQuerySQLException { + ConnectionImpl connectionSpy = Mockito.spy(connection); + List parameters = new ArrayList<>(); + Map labels = new HashMap<>(); + doReturn(false).when(connectionSpy).isFastQuerySupported(); + com.google.api.services.bigquery.model.JobStatistics jobStatistics = + new com.google.api.services.bigquery.model.JobStatistics(); + com.google.api.services.bigquery.model.Job jobResponseMock = + new com.google.api.services.bigquery.model.Job() + .setJobReference(QUERY_JOB.toPb()) + .setId(JOB) + .setStatus(new com.google.api.services.bigquery.model.JobStatus().setState("DONE")) + .setStatistics(jobStatistics); + + doReturn(jobResponseMock) + .when(connectionSpy) + .createQueryJob(SQL_QUERY, connectionSettings, parameters, labels); + doReturn(GET_QUERY_RESULTS_RESPONSE) + .when(connectionSpy) + .getQueryResultsFirstPage(any(JobId.class)); + doReturn(BQ_RS_MOCK_RES) + .when(connectionSpy) + .getResultSet(any(GetQueryResultsResponse.class), any(JobId.class), any(String.class)); + BigQueryResultSet res = connectionSpy.executeSelect(SQL_QUERY, parameters, labels); + assertEquals(res.getTotalRows(), 2); + assertEquals(QUERY_SCHEMA, res.getSchema()); + verify(connectionSpy, times(1)) + .getResultSet(any(GetQueryResultsResponse.class), any(JobId.class), any(String.class)); + } + + @Test + public void testGetSubsequentQueryResultsWithJob() { + ConnectionImpl connectionSpy = Mockito.spy(connection); + JobId jobId = mock(JobId.class); + BigQueryResultSetStats bqRsStats = mock(BigQueryResultSetStats.class); + doReturn(true) + .when(connectionSpy) + .useReadAPI(any(Long.class), any(Long.class), any(Schema.class)); + doReturn(BQ_RS_MOCK_RES) + .when(connectionSpy) + .highThroughPutRead( + any(TableId.class), + any(Long.class), + any(Schema.class), + any(BigQueryResultSetStats.class)); + + doReturn(TABLE_NAME).when(connectionSpy).getDestinationTable(any(JobId.class)); + doReturn(bqRsStats).when(connectionSpy).getBigQueryResultSetStats(any(JobId.class)); + BigQueryResultSet res = + connectionSpy.getSubsequentQueryResultsWithJob( + 10000L, 100L, jobId, GET_QUERY_RESULTS_RESPONSE); + assertEquals(res.getTotalRows(), 2); + assertEquals(QUERY_SCHEMA, res.getSchema()); + verify(connectionSpy, times(1)) + .getSubsequentQueryResultsWithJob(10000L, 100L, jobId, GET_QUERY_RESULTS_RESPONSE); + } + + @Test + public void testGetPageCacheSize() { + ConnectionImpl connectionSpy = Mockito.spy(connection); + // number of cached pages should be within a range + assertTrue(connectionSpy.getPageCacheSize(10000L, 100, QUERY_SCHEMA) >= 3); + assertTrue(connectionSpy.getPageCacheSize(100000000L, 100, QUERY_SCHEMA) <= 20); + verify(connectionSpy, times(2)) + .getPageCacheSize(any(Long.class), any(Long.class), any(Schema.class)); + } } From 69de2e8f108c46668ddb34ed269e8e924abd91d2 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Mon, 21 Mar 2022 17:04:58 +0530 Subject: [PATCH 170/277] fixed getString to handle LocalDateTime --- .../com/google/cloud/bigquery/BigQueryResultSetImpl.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index ff8b1c780..e21102faf 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -23,6 +23,7 @@ import java.sql.SQLException; import java.sql.Time; import java.sql.Timestamp; +import java.time.LocalDateTime; import java.time.LocalTime; import java.time.ZoneId; import java.util.Map; @@ -154,10 +155,12 @@ public String getString(String fieldName) throws SQLException { Object currentVal = curTuple.x().get(fieldName); if (currentVal == null) { return null; - } - if (currentVal instanceof JsonStringArrayList) { // arrays + } else if (currentVal instanceof JsonStringArrayList) { // arrays JsonStringArrayList jsnAry = (JsonStringArrayList) currentVal; return jsnAry.toString(); + } else if (currentVal instanceof LocalDateTime) { + LocalDateTime dateTime = (LocalDateTime) currentVal; + return dateTime.toString(); } else { Text textVal = (Text) currentVal; return textVal.toString(); From 4fa2f9668ac66d91f3110c92bb4e3bbde124164e Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 24 Mar 2022 11:33:25 +0530 Subject: [PATCH 171/277] Added ConnImplBenchmark --- .../ConnImplBenchmark.java | 181 ++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java diff --git a/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java b/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java new file mode 100644 index 000000000..8eef2e770 --- /dev/null +++ b/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java @@ -0,0 +1,181 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import java.io.IOException; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +@Fork(value = 1) +@BenchmarkMode(Mode.AverageTime) +@Warmup(iterations = 0) +@Measurement(iterations = 3) +@State(Scope.Benchmark) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +public class ConnImplBenchmark { + @Param({"500000", "1000000", "10000000", "100000000"}) // 500K, 1M, 10M, and 100M + public int rowLimit; + + private ConnectionSettings connectionSettingsReadAPIEnabled, connectionSettingsReadAPIDisabled; + private long numBuffRows = 100000L; + private final String DATASET = "bigquery_test_dataset"; + private final String QUERY = "SELECT * FROM tlc_yellow_trips_2017_stephwang LIMIT %s"; + + @Setup + public void setUp() throws IOException { + java.util.logging.Logger.getGlobal().setLevel(Level.ALL); + ReadClientConnectionConfiguration clientConnectionConfiguration; + + clientConnectionConfiguration = + ReadClientConnectionConfiguration.newBuilder() + .setTotalToPageRowCountRatio(10L) + .setMinResultSize(200000L) + .setBufferSize(numBuffRows) + .build(); + + connectionSettingsReadAPIEnabled = + ConnectionSettings.newBuilder() + .setDefaultDataset(DatasetId.of(DATASET)) + .setNumBufferedRows(numBuffRows) // page size + .setPriority( + QueryJobConfiguration.Priority + .INTERACTIVE) // DEFAULT VALUE - so that isFastQuerySupported returns false + .setReadClientConnectionConfiguration(clientConnectionConfiguration) + .setUseReadAPI(true) // enable read api + .build(); + connectionSettingsReadAPIDisabled = + ConnectionSettings.newBuilder() + .setDefaultDataset(DatasetId.of(DATASET)) + .setNumBufferedRows(numBuffRows) // page size + .setPriority( + QueryJobConfiguration.Priority + .INTERACTIVE) // so that isFastQuerySupported returns false + .setReadClientConnectionConfiguration(clientConnectionConfiguration) + .setUseReadAPI(false) // disable read api + .build(); + } + + @Benchmark + public void iterateRecordsUsingReadAPI(Blackhole blackhole) + throws InterruptedException, BigQuerySQLException { + Connection connectionReadAPIEnabled = + BigQueryOptions.getDefaultInstance() + .getService() + .createConnection(connectionSettingsReadAPIEnabled); + String selectQuery = String.format(QUERY, rowLimit); + long hash = 0L; + try { + BigQueryResultSet bigQueryResultSet = connectionReadAPIEnabled.executeSelect(selectQuery); + hash = getResultHash(bigQueryResultSet); + } catch (Exception e) { + e.printStackTrace(); + } finally { + connectionReadAPIEnabled.cancel(); // IMP to kill the bg workers + } + blackhole.consume(hash); + } + + @Benchmark + public void iterateRecordsWithoutUsingReadAPI(Blackhole blackhole) + throws InterruptedException, BigQuerySQLException { + Connection connectionReadAPIDisabled = + BigQueryOptions.getDefaultInstance() + .getService() + .createConnection(connectionSettingsReadAPIDisabled); + String selectQuery = String.format(QUERY, rowLimit); + long hash = 0L; + try { + BigQueryResultSet bigQueryResultSet = connectionReadAPIDisabled.executeSelect(selectQuery); + hash = getResultHash(bigQueryResultSet); + } catch (Exception e) { + e.printStackTrace(); + } finally { + connectionReadAPIDisabled.cancel(); // IMP to kill the bg workers + } + blackhole.consume(hash); + } + + // Hashes all the 20 columns of all the rows + private long getResultHash(BigQueryResultSet bigQueryResultSet) throws SQLException { + ResultSet rs = bigQueryResultSet.getResultSet(); + long hash = 0L; + int cnt = 0; + System.out.print("\n Running"); + while (rs.next()) { + hash += rs.getString("vendor_id") == null ? 0 : rs.getString("vendor_id").hashCode(); + hash += + rs.getString("pickup_datetime") == null ? 0 : rs.getString("pickup_datetime").hashCode(); + hash += + rs.getString("dropoff_datetime") == null + ? 0 + : rs.getString("dropoff_datetime").hashCode(); + hash += rs.getLong("passenger_count"); + hash += rs.getDouble("trip_distance"); + hash += rs.getDouble("pickup_longitude"); + hash += rs.getDouble("pickup_latitude"); + hash += rs.getString("rate_code") == null ? 0 : rs.getString("rate_code").hashCode(); + hash += + rs.getString("store_and_fwd_flag") == null + ? 0 + : rs.getString("store_and_fwd_flag").hashCode(); + hash += rs.getDouble("dropoff_longitude"); + hash += rs.getDouble("dropoff_latitude"); + hash += rs.getString("payment_type") == null ? 0 : rs.getString("payment_type").hashCode(); + hash += rs.getDouble("fare_amount"); + hash += rs.getDouble("extra"); + hash += rs.getDouble("mta_tax"); + hash += rs.getDouble("tip_amount"); + hash += rs.getDouble("tolls_amount"); + hash += rs.getDouble("imp_surcharge"); + hash += rs.getDouble("total_amount"); + hash += + rs.getString("pickup_location_id") == null + ? 0 + : rs.getString("pickup_location_id").hashCode(); + hash += + rs.getString("dropoff_location_id") == null + ? 0 + : rs.getString("dropoff_location_id").hashCode(); + if (++cnt % 100000 == 0) { // just to indicate the progress while long running benchmarks + System.out.print("."); + } + } + return hash; + } + + public static void main(String[] args) throws Exception { + Options opt = new OptionsBuilder().include(ConnImplBenchmark.class.getSimpleName()).build(); + new Runner(opt).run(); + } +} From 5eaa7b6c0fbaf32a95490ba772257c9e193c27e5 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 24 Mar 2022 11:34:35 +0530 Subject: [PATCH 172/277] Added instructions for running ConnImplBenchmark --- benchmark/README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/benchmark/README.md b/benchmark/README.md index 41e9c2fda..d1a1ae157 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -19,3 +19,10 @@ To run a benchmark jar, run the following command cd benchmark java -jar target/benchmark.jar ``` + +To run ConnImplBenchmark, run the following command +``` +# Run from benchmark directory + cd benchmark + java -jar target/benchmark.jar com.google.cloud.bigquery.ConnImplBenchmark +``` From a7f20ed62c1a1275c900dafa8fce33ef356e93c3 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 29 Mar 2022 11:24:02 +0530 Subject: [PATCH 173/277] Updated Java Docs --- .../main/java/com/google/cloud/bigquery/BigQueryResultSet.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java index c36e95112..15fc7ede0 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java @@ -25,7 +25,8 @@ public interface BigQueryResultSet { /** * Returns the total number of rows in the complete result set, which can be more than the number - * of rows in the first page of results. + * of rows in the first page of results. This might return -1 if the query is long running and the + * job is not complete at the time this object is returned. */ long getTotalRows(); From 1218d2f634d9c6c7f0ec3c481cd3c8b1e9c1b477 Mon Sep 17 00:00:00 2001 From: stephwang Date: Tue, 29 Mar 2022 16:44:07 -0400 Subject: [PATCH 174/277] add IT TODOs --- .../cloud/bigquery/it/ITBigQueryTest.java | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 749616eba..a7f31fb01 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -950,6 +950,7 @@ public void testCreateTableWithRangePartitioning() { } } + /* TODO(prasmish): replicate this test case for executeSelect on the relevant part */ @Test public void testJsonType() throws InterruptedException { String tableName = "test_create_table_jsontype"; @@ -1057,6 +1058,7 @@ public void testJsonType() throws InterruptedException { } } + /* TODO(prasmish): replicate this test case for executeSelect on the relevant part */ @Test public void testIntervalType() throws InterruptedException { String tableName = "test_create_table_intervaltype"; @@ -1938,6 +1940,7 @@ public void testInsertAllWithErrors() { assertTrue(bigquery.delete(TableId.of(DATASET, tableName))); } + /* TODO(prasmish): replicate the entire test case for executeSelect */ @Test public void testListAllTableData() { Page rows = bigquery.listTableData(TABLE_ID); @@ -2273,6 +2276,7 @@ public void testAuthorizeDataset() { assertEquals(sharedDatasetAcl, updatedDataset.getAcl()); } + /* TODO(prasmish): replicate the entire test case for executeSelect */ @Test public void testSingleStatementsQueryException() throws InterruptedException { String invalidQuery = @@ -2289,6 +2293,7 @@ public void testSingleStatementsQueryException() throws InterruptedException { } } + /* TODO(prasmish): replicate the entire test case for executeSelect */ @Test public void testMultipleStatementsQueryException() throws InterruptedException { String invalidQuery = @@ -2307,6 +2312,7 @@ public void testMultipleStatementsQueryException() throws InterruptedException { } } + /* TODO(prasmish): replicate the entire test case for executeSelect */ @Test public void testQuery() throws InterruptedException { String query = "SELECT TimestampField, StringField, BooleanField FROM " + TABLE_ID.getTable(); @@ -2339,6 +2345,7 @@ public void testQuery() throws InterruptedException { assertNotNull(statistics.getQueryPlan()); } + /* TODO(prasmish): replicate the entire test case for executeSelect */ @Test public void testQueryTimeStamp() throws InterruptedException { String query = "SELECT TIMESTAMP '2022-01-24T23:54:25.095574Z'"; @@ -2373,6 +2380,7 @@ public void testQueryTimeStamp() throws InterruptedException { } } + /* TODO(prasmish): replicate the entire test case for executeSelect */ @Test public void testQueryCaseInsensitiveSchemaFieldByGetName() throws InterruptedException { String query = "SELECT TimestampField, StringField, BooleanField FROM " + TABLE_ID.getTable(); @@ -2401,6 +2409,7 @@ public void testQueryCaseInsensitiveSchemaFieldByGetName() throws InterruptedExc assertEquals(2, rowCount); } + /* TODO(prasmish): replicate bigquery.query part of the test case for executeSelect - modify this test case */ @Test public void testQueryExternalHivePartitioningOptionAutoLayout() throws InterruptedException { String tableName = "test_queryexternalhivepartition_autolayout_table"; @@ -2435,6 +2444,7 @@ public void testQueryExternalHivePartitioningOptionAutoLayout() throws Interrupt assertTrue(bigquery.delete(tableId)); } + /* TODO(prasmish): replicate bigquery.query part of the test case for executeSelect - modify this test case */ @Test public void testQueryExternalHivePartitioningOptionCustomLayout() throws InterruptedException { String tableName = "test_queryexternalhivepartition_customlayout_table"; @@ -2848,6 +2858,7 @@ public void testExecuteSelectArrayOfStruct() throws SQLException { assertEquals(10, arrayOfStructFieldValue.get(1).getRecordValue().get(1).getLongValue()); } + /* TODO(prasmish): replicate the entire test case for executeSelect */ @Test public void testFastQueryMultipleRuns() throws InterruptedException { String query = @@ -2880,6 +2891,7 @@ public void testFastQueryMultipleRuns() throws InterruptedException { assertFalse(result2.hasNextPage()); } + /* TODO(prasmish): replicate the entire test case for executeSelect */ @Test public void testFastQuerySinglePageDuplicateRequestIds() throws InterruptedException { String query = @@ -2909,6 +2921,7 @@ public void testFastQuerySinglePageDuplicateRequestIds() throws InterruptedExcep assertFalse(result2.hasNextPage()); } + /* TODO(prasmish): replicate the entire test case for executeSelect */ @Test public void testFastSQLQuery() throws InterruptedException { String query = @@ -2938,6 +2951,7 @@ public void testFastSQLQuery() throws InterruptedException { } } + /* TODO(prasmish): replicate the entire test case for executeSelect */ @Test public void testFastSQLQueryMultiPage() throws InterruptedException { String query = @@ -3064,6 +3078,7 @@ public void testFastQuerySlowDDL() throws InterruptedException { } } + /* TODO(prasmish): replicate the entire test case for executeSelect */ @Test public void testFastQueryHTTPException() throws InterruptedException { String queryInvalid = @@ -3166,6 +3181,7 @@ public void testDmlStatistics() throws InterruptedException { assertEquals(2L, statistics.getDmlStats().getUpdatedRowCount().longValue()); } + /* TODO(prasmish): replicate the entire test case for executeSelect */ @Test public void testTransactionInfo() throws InterruptedException { String tableName = TABLE_ID_FASTQUERY.getTable(); @@ -3187,6 +3203,7 @@ public void testTransactionInfo() throws InterruptedException { } } + /* TODO(prasmish): replicate the entire test case for executeSelect */ @Test public void testScriptStatistics() throws InterruptedException { String script = @@ -3267,6 +3284,7 @@ public void testQueryParameterModeWithDryRun() { assertNotNull(statistics.getTotalBytesProcessed()); } + /* TODO(prasmish): replicate relevant parts of the test case for executeSelect */ @Test public void testPositionalQueryParameters() throws InterruptedException { String query = @@ -3337,6 +3355,7 @@ public void testPositionalQueryParameters() throws InterruptedException { } } + /* TODO(prasmish): replicate relevant parts of the test case for executeSelect */ @Test public void testNamedQueryParameters() throws InterruptedException { String query = @@ -3359,6 +3378,7 @@ public void testNamedQueryParameters() throws InterruptedException { assertEquals(2, Iterables.size(result.getValues())); } + /* TODO(prasmish): replicate relevant parts of the test case for executeSelect */ @Test public void testStructNamedQueryParameters() throws InterruptedException { QueryParameterValue booleanValue = QueryParameterValue.bool(true); @@ -3413,6 +3433,7 @@ private static void assertsFieldValue(FieldValue record) { assertEquals("test-stringField", record.getRecordValue().get("stringField").getStringValue()); } + /* TODO(prasmish): replicate relevant parts of the test case for executeSelect */ @Test public void testNestedStructNamedQueryParameters() throws InterruptedException { QueryParameterValue booleanValue = QueryParameterValue.bool(true); @@ -3456,6 +3477,7 @@ public void testNestedStructNamedQueryParameters() throws InterruptedException { } } + /* TODO(prasmish): replicate relevant parts of the test case for executeSelect */ @Test public void testBytesParameter() throws Exception { String query = "SELECT BYTE_LENGTH(@p) AS length"; @@ -3745,6 +3767,7 @@ public void testCopyJobWithLabels() throws InterruptedException { assertTrue(remoteTable.delete()); } + /* TODO(prasmish): replicate the entire test case for executeSelect */ @Test public void testQueryJob() throws InterruptedException, TimeoutException { String tableName = "test_query_job_table"; @@ -3789,6 +3812,7 @@ public void testQueryJob() throws InterruptedException, TimeoutException { assertNotNull(statistics.getQueryPlan()); } + /* TODO(prasmish): replicate the entire test case for executeSelect */ @Test public void testQueryJobWithConnectionProperties() throws InterruptedException { String tableName = "test_query_job_table_connection_properties"; @@ -3808,6 +3832,7 @@ public void testQueryJobWithConnectionProperties() throws InterruptedException { assertTrue(bigquery.delete(destinationTable)); } + /* TODO(prasmish): replicate the entire test case for executeSelect */ @Test public void testQueryJobWithLabels() throws InterruptedException, TimeoutException { String tableName = "test_query_job_table"; @@ -3831,6 +3856,7 @@ public void testQueryJobWithLabels() throws InterruptedException, TimeoutExcepti } } + /* TODO(prasmish): replicate the entire test case for executeSelect */ @Test public void testQueryJobWithRangePartitioning() throws InterruptedException { String tableName = "test_query_job_table_rangepartitioning"; From 5675160c40514989c90a78a3dbb343f335810f1a Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Mon, 4 Apr 2022 10:18:14 +0530 Subject: [PATCH 175/277] Fixed getInt for JSONField types (Int returned as Text) --- .../com/google/cloud/bigquery/BigQueryResultSetImpl.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index e21102faf..22307996d 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -198,7 +198,13 @@ public int getInt(String fieldName) throws SQLException { throw new SQLException(String.format("Field %s not found", fieldName)); } Object curVal = curTuple.x().get(fieldName); - return curVal == null ? 0 : ((BigDecimal) curVal).intValue(); + if (curVal == null) { + return 0; + } + if (curVal instanceof Text) { // parse from text to int + return Integer.parseInt(((Text) curVal).toString()); + } + return ((BigDecimal) curVal).intValue(); } } From e15c1cf1b5daf1a352c9186364e5848b3bfc2dc5 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Mon, 4 Apr 2022 10:41:22 +0530 Subject: [PATCH 176/277] Added JsonType tests --- .../cloud/bigquery/it/ITNightlyBigQueryTest.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java index a1ee848b7..80ab94fe3 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java @@ -74,8 +74,8 @@ public class ITNightlyBigQueryTest { private static final String QUERY = String.format( "select StringField, GeographyField, BooleanField, BigNumericField, IntegerField, NumericField, BytesField, TimestampField, TimeField, DateField, " - + "IntegerArrayField, RecordField.BooleanField, RecordField.StringField , JSONField from %s order by IntegerField asc LIMIT %s", - TABLE, LIMIT_RECS); + + "IntegerArrayField, RecordField.BooleanField, RecordField.StringField , JSONField, JSONField.hello, JSONField.id from %s.%s order by IntegerField asc LIMIT %s", + DATASET, TABLE, LIMIT_RECS); private static final Schema BQ_SCHEMA = Schema.of( @@ -196,6 +196,8 @@ public void testIterateAndOrder() throws SQLException { Connection connection = bigquery.createConnection(connectionSettings); BigQueryResultSet bigQueryResultSet = connection.executeSelect(QUERY); + System.out.println(QUERY); + logger.log(Level.INFO, "Query used: {0}", QUERY); ResultSet rs = bigQueryResultSet.getResultSet(); int cnt = 0; @@ -222,6 +224,9 @@ public void testIterateAndOrder() throws SQLException { assertNull(rs.getString("JSONField")); assertFalse(rs.getBoolean("BooleanField_1")); assertNull(rs.getString("StringField_1")); + assertNull(rs.getString("hello")); // equivalent of testJsonType + assertEquals(0, rs.getInt("id")); + } else { // remaining rows are supposed to be non null assertNotNull(rs.getString("StringField")); assertNotNull(rs.getString("GeographyField")); @@ -241,6 +246,10 @@ public void testIterateAndOrder() throws SQLException { // check the order of the records assertTrue(prevIntegerFieldVal < rs.getInt("IntegerField")); prevIntegerFieldVal = rs.getInt("IntegerField"); + + // Testing JSON type + assertEquals("\"world\"", rs.getString("hello")); // BQ stores the value as "world" + assertEquals(100, rs.getInt("id")); } } assertEquals(LIMIT_RECS, cnt); @@ -340,7 +349,7 @@ private static Map getNextRow() { row.put("TimeField", "12:11:35"); row.put("DateField", "2022-01-01"); row.put("DateTimeField", "0001-01-01 00:00:00"); - row.put("JSONField", "{\"hello\":\"world\"}"); + row.put("JSONField", "{\"hello\":\"world\",\"id\":100}"); row.put("IntervalField", "10000-0 3660000 87840000:0:0"); return row; } From f407e1b2e403f2be8bff7294f9fcf728a4f64498 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Mon, 4 Apr 2022 18:02:08 +0530 Subject: [PATCH 177/277] Added test - testForAllDataTypeValues --- .../bigquery/it/ITNightlyBigQueryTest.java | 94 ++++++++++++++----- 1 file changed, 68 insertions(+), 26 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java index 80ab94fe3..7ed222a19 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java @@ -45,12 +45,19 @@ import com.google.cloud.bigquery.TableId; import com.google.cloud.bigquery.TableInfo; import com.google.cloud.bigquery.testing.RemoteBigQueryHelper; +import com.google.common.io.BaseEncoding; import java.io.IOException; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Time; +import java.time.LocalTime; +import java.time.ZoneId; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.TimeZone; import java.util.concurrent.ExecutionException; import java.util.logging.Level; import java.util.logging.Logger; @@ -65,16 +72,19 @@ public class ITNightlyBigQueryTest { private static final Logger logger = Logger.getLogger(ITNightlyBigQueryTest.class.getName()); private static final String DATASET = RemoteBigQueryHelper.generateDatasetName(); private static final String TABLE = "TEMP_RS_TEST_TABLE"; + private static final byte[] BYTES = "TestByteValue".getBytes(StandardCharsets.UTF_8); + private static final String BYTES_BASE64 = BaseEncoding.base64().encode(BYTES); // Script will populate NUM_BATCHES*REC_PER_BATCHES number of records (eg: 100*10000 = 1M) - private static final int NUM_BATCHES = 55; + private static final int NUM_BATCHES = 35; private static final int REC_PER_BATCHES = 10000; - private static final int LIMIT_RECS = 500000; + private static final int LIMIT_RECS = 300000; private static int rowCnt = 0; private static BigQuery bigquery; private static final String QUERY = String.format( - "select StringField, GeographyField, BooleanField, BigNumericField, IntegerField, NumericField, BytesField, TimestampField, TimeField, DateField, " - + "IntegerArrayField, RecordField.BooleanField, RecordField.StringField , JSONField, JSONField.hello, JSONField.id from %s.%s order by IntegerField asc LIMIT %s", + "select StringField, GeographyField, BooleanField, BigNumericField, IntegerField, NumericField, BytesField, " + + "TimestampField, TimeField, DateField, IntegerArrayField, RecordField.BooleanField, RecordField.StringField ," + + " JSONField, JSONField.hello, JSONField.id from %s.%s order by IntegerField asc LIMIT %s", DATASET, TABLE, LIMIT_RECS); private static final Schema BQ_SCHEMA = @@ -137,10 +147,6 @@ public class ITNightlyBigQueryTest { .setMode(Field.Mode.NULLABLE) .setDescription("DateDescription") .build(), - Field.newBuilder("DateTimeField", StandardSQLTypeName.DATETIME) - .setMode(Field.Mode.NULLABLE) - .setDescription("DateTimeDescription") - .build(), Field.newBuilder("JSONField", StandardSQLTypeName.JSON) .setMode(Field.Mode.NULLABLE) .setDescription("JSONFieldDescription") @@ -196,15 +202,13 @@ public void testIterateAndOrder() throws SQLException { Connection connection = bigquery.createConnection(connectionSettings); BigQueryResultSet bigQueryResultSet = connection.executeSelect(QUERY); - System.out.println(QUERY); logger.log(Level.INFO, "Query used: {0}", QUERY); ResultSet rs = bigQueryResultSet.getResultSet(); int cnt = 0; int prevIntegerFieldVal = 0; while (rs.next()) { - ++cnt; - if (cnt == 1) { // first row is supposed to be null + if (cnt == 0) { // first row is supposed to be null assertNull(rs.getString("StringField")); assertNull(rs.getString("GeographyField")); Object intAryField = rs.getObject("IntegerArrayField"); @@ -247,12 +251,55 @@ public void testIterateAndOrder() throws SQLException { assertTrue(prevIntegerFieldVal < rs.getInt("IntegerField")); prevIntegerFieldVal = rs.getInt("IntegerField"); - // Testing JSON type - assertEquals("\"world\"", rs.getString("hello")); // BQ stores the value as "world" - assertEquals(100, rs.getInt("id")); + testForAllDataTypeValues(rs, cnt); // asserts the value of each row } + ++cnt; } - assertEquals(LIMIT_RECS, cnt); + assertEquals(LIMIT_RECS, cnt); // all the records were retrieved + } + + // asserts the value of each row + private static void testForAllDataTypeValues(ResultSet rs, int cnt) throws SQLException { + // Testing JSON type + assertEquals("\"world\"", rs.getString("hello")); // BQ stores the value as "world" + assertEquals(100, rs.getInt("id")); + assertEquals("{\"hello\":\"world\",\"id\":100}", rs.getString("JSONField")); + + // String and Geography types + assertEquals(String.format("String Val %s", cnt), rs.getString("StringField")); + assertEquals("POINT(1 2)", rs.getString("GeographyField")); + + // Array type tests + JsonStringArrayList ary = (JsonStringArrayList) rs.getObject("IntegerArrayField"); + assertEquals(3, ary.size()); + assertEquals(1, ary.get(0).intValue()); + assertEquals(2, ary.get(1).intValue()); + assertEquals(3, ary.get(2).intValue()); + + // BigNumeric, int and Numeric + assertTrue(10000000L + cnt == rs.getDouble("BigNumericField")); + assertEquals(1 + cnt, rs.getInt("IntegerField")); + assertEquals(100 + cnt, rs.getLong("NumericField")); + // Test Byte field + assertEquals("TestByteValue", new String(rs.getBytes("BytesField"), StandardCharsets.UTF_8)); + + // Struct Fields + assertFalse(rs.getBoolean("BooleanField_1")); + assertEquals(String.format("Str Val %s", cnt), rs.getString("StringField_1")); + + // Timestamp, Time, DateTime and Date fields + assertEquals(1649064795000L, rs.getTimestamp("TimestampField").getTime()); + assertEquals( + java.sql.Date.valueOf("2022-01-01").toString(), rs.getDate("DateField").toString()); + // Time is represented independent of a specific date and timezone. For example a 12:11:35 (GMT) + // is returned as + // 17:11:35 (GMT+5:30) . So we need to adjust the offset + int offset = + TimeZone.getTimeZone(ZoneId.systemDefault()) + .getOffset(new java.util.Date().getTime()); // offset in seconds + assertEquals( + Time.valueOf(LocalTime.of(12, 11, 35)).getTime() + offset, + rs.getTime("TimeField").getTime()); } private static void populateTestRecords(String datasetName, String tableName) { @@ -277,17 +324,13 @@ private static void addBatchRecords(TableId tableId) { if (response.hasErrors()) { // If any of the insertions failed, this lets you inspect the errors for (Map.Entry> entry : response.getInsertErrors().entrySet()) { - logger.log( - Level.WARNING, - "Exception while adding records {0}", - entry.getValue()); // gracefully log it, it's okay if a few batches fail during IT + logger.log(Level.WARNING, "Exception while adding records {0}", entry.getValue()); } + fail("Response has errors"); } } catch (BigQueryException e) { - logger.log( - Level.WARNING, - "Exception while adding records {0}", - e); // gracefully log it, it's okay if a few batches fail during IT + logger.log(Level.WARNING, "Exception while adding records {0}", e); + fail("Error in addBatchRecords"); } } @@ -337,18 +380,17 @@ private static Map getNextRow() { structVal.put("StringField", "Str Val " + rowCnt); structVal.put("BooleanField", false); row.put("RecordField", structVal); // struct - row.put("TimestampField", "2022-01-01 01:01:01"); + row.put("TimestampField", "2022-04-04 15:03:15.000 +05:30"); row.put("StringField", "String Val " + rowCnt); row.put("IntegerArrayField", new int[] {1, 2, 3}); row.put("BooleanField", true); - row.put("BytesField", "DQ4KDQ=="); + row.put("BytesField", BYTES_BASE64); row.put("IntegerField", 1 + rowCnt); row.put("GeographyField", "POINT(1 2)"); row.put("NumericField", 100 + rowCnt); row.put("BigNumericField", 10000000L + rowCnt); row.put("TimeField", "12:11:35"); row.put("DateField", "2022-01-01"); - row.put("DateTimeField", "0001-01-01 00:00:00"); row.put("JSONField", "{\"hello\":\"world\",\"id\":100}"); row.put("IntervalField", "10000-0 3660000 87840000:0:0"); return row; From 52f23e1d62f3f53907d137090ffe30b1efe0d65d Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 5 Apr 2022 11:26:11 +0530 Subject: [PATCH 178/277] Added translateAndThrowBigQuerySQLException --- .../java/com/google/cloud/bigquery/BigQueryException.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryException.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryException.java index 818c0beb1..c42ff6327 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryException.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryException.java @@ -138,4 +138,9 @@ static BaseServiceException translateAndThrow(ExecutionException ex) { static BaseServiceException translateAndThrow(Exception ex) { throw new BigQueryException(UNKNOWN_CODE, ex.getMessage(), ex.getCause()); } + + static BaseServiceException translateAndThrowBigQuerySQLException(BigQueryException e) + throws BigQuerySQLException { + throw new BigQuerySQLException(e.getMessage(), e, e.getErrors()); + } } From f77a79109d174015fd498e3090fbd46253d14fba Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 5 Apr 2022 11:26:40 +0530 Subject: [PATCH 179/277] Added Exception Handling for executeSelect methods --- .../google/cloud/bigquery/ConnectionImpl.java | 54 +++++++++++-------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 9d20eeecd..1118d11e1 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -143,18 +143,22 @@ public BigQueryDryRunResult dryRun(String sql) throws BigQuerySQLException { @BetaApi @Override public BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException { - // use jobs.query if all the properties of connectionSettings are supported - if (isFastQuerySupported()) { - String projectId = bigQueryOptions.getProjectId(); - QueryRequest queryRequest = createQueryRequest(connectionSettings, sql, null, null); - return queryRpc(projectId, queryRequest); + try { + // use jobs.query if all the properties of connectionSettings are supported + if (isFastQuerySupported()) { + String projectId = bigQueryOptions.getProjectId(); + QueryRequest queryRequest = createQueryRequest(connectionSettings, sql, null, null); + return queryRpc(projectId, queryRequest); + } + // use jobs.insert otherwise + com.google.api.services.bigquery.model.Job queryJob = + createQueryJob(sql, connectionSettings, null, null); + JobId jobId = JobId.fromPb(queryJob.getJobReference()); + GetQueryResultsResponse firstPage = getQueryResultsFirstPage(jobId); + return getResultSet(firstPage, jobId, sql); + } catch (BigQueryException e) { + throw new BigQuerySQLException(e.getMessage(), e, e.getErrors()); } - // use jobs.insert otherwise - com.google.api.services.bigquery.model.Job queryJob = - createQueryJob(sql, connectionSettings, null, null); - JobId jobId = JobId.fromPb(queryJob.getJobReference()); - GetQueryResultsResponse firstPage = getQueryResultsFirstPage(jobId); - return getResultSet(firstPage, jobId, sql); } @VisibleForTesting @@ -195,19 +199,23 @@ BigQueryResultSet getResultSet(GetQueryResultsResponse firstPage, JobId jobId, S public BigQueryResultSet executeSelect( String sql, List parameters, Map labels) throws BigQuerySQLException { - // use jobs.query if possible - if (isFastQuerySupported()) { - final String projectId = bigQueryOptions.getProjectId(); - final QueryRequest queryRequest = - createQueryRequest(connectionSettings, sql, parameters, labels); - return queryRpc(projectId, queryRequest); + try { + // use jobs.query if possible + if (isFastQuerySupported()) { + final String projectId = bigQueryOptions.getProjectId(); + final QueryRequest queryRequest = + createQueryRequest(connectionSettings, sql, parameters, labels); + return queryRpc(projectId, queryRequest); + } + // use jobs.insert otherwise + com.google.api.services.bigquery.model.Job queryJob = + createQueryJob(sql, connectionSettings, parameters, labels); + JobId jobId = JobId.fromPb(queryJob.getJobReference()); + GetQueryResultsResponse firstPage = getQueryResultsFirstPage(jobId); + return getResultSet(firstPage, jobId, sql); + } catch (BigQueryException e) { + throw new BigQuerySQLException(e.getMessage(), e, e.getErrors()); } - // use jobs.insert otherwise - com.google.api.services.bigquery.model.Job queryJob = - createQueryJob(sql, connectionSettings, parameters, labels); - JobId jobId = JobId.fromPb(queryJob.getJobReference()); - GetQueryResultsResponse firstPage = getQueryResultsFirstPage(jobId); - return getResultSet(firstPage, jobId, sql); } static class EndOfFieldValueList From dece7acbca285b1f3d44569421297a57aa13e540 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 5 Apr 2022 11:27:11 +0530 Subject: [PATCH 180/277] Added testInvalidQuery --- .../bigquery/it/ITNightlyBigQueryTest.java | 58 +++++++++++++------ 1 file changed, 40 insertions(+), 18 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java index 7ed222a19..e160a21db 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java @@ -27,6 +27,7 @@ import com.google.cloud.bigquery.BigQueryError; import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.BigQueryResultSet; +import com.google.cloud.bigquery.BigQuerySQLException; import com.google.cloud.bigquery.Connection; import com.google.cloud.bigquery.ConnectionSettings; import com.google.cloud.bigquery.Dataset; @@ -86,6 +87,9 @@ public class ITNightlyBigQueryTest { + "TimestampField, TimeField, DateField, IntegerArrayField, RecordField.BooleanField, RecordField.StringField ," + " JSONField, JSONField.hello, JSONField.id from %s.%s order by IntegerField asc LIMIT %s", DATASET, TABLE, LIMIT_RECS); + private static final String INVALID_QUERY = + String.format( + "select into %s.%s order by IntegerField asc LIMIT %s", DATASET, TABLE, LIMIT_RECS); private static final Schema BQ_SCHEMA = Schema.of( @@ -182,25 +186,22 @@ public static void afterClass() throws ExecutionException, InterruptedException } @Test - public void testIterateAndOrder() throws SQLException { - ReadClientConnectionConfiguration clientConnectionConfiguration = - ReadClientConnectionConfiguration.newBuilder() - .setTotalToPageRowCountRatio(10L) - .setMinResultSize(200000L) - .setBufferSize(10000L) - .build(); - ConnectionSettings connectionSettings = - ConnectionSettings.newBuilder() - .setDefaultDataset(DatasetId.of(DATASET)) - .setNumBufferedRows(10000L) // page size - .setPriority( - QueryJobConfiguration.Priority - .INTERACTIVE) // so that isFastQuerySupported returns false - .setReadClientConnectionConfiguration(clientConnectionConfiguration) - .setUseReadAPI(true) - .build(); - Connection connection = bigquery.createConnection(connectionSettings); + public void testInvalidQuery() throws BigQuerySQLException { + Connection connection = getConnection(); + try { + BigQueryResultSet bigQueryResultSet = connection.executeSelect(INVALID_QUERY); + fail("BigQuerySQLException was expected"); + } catch (BigQuerySQLException ex) { + assertNotNull(ex.getMessage()); + assertTrue(ex.getMessage().toLowerCase().contains("unexpected keyword into")); + } finally { + connection.cancel(); + } + } + @Test + public void testIterateAndOrder() throws SQLException { + Connection connection = getConnection(); BigQueryResultSet bigQueryResultSet = connection.executeSelect(QUERY); logger.log(Level.INFO, "Query used: {0}", QUERY); ResultSet rs = bigQueryResultSet.getResultSet(); @@ -256,6 +257,7 @@ public void testIterateAndOrder() throws SQLException { ++cnt; } assertEquals(LIMIT_RECS, cnt); // all the records were retrieved + connection.cancel(); } // asserts the value of each row @@ -373,6 +375,26 @@ public static void deleteDataset(String datasetName) { } } + private Connection getConnection() { + ReadClientConnectionConfiguration clientConnectionConfiguration = + ReadClientConnectionConfiguration.newBuilder() + .setTotalToPageRowCountRatio(10L) + .setMinResultSize(200000L) + .setBufferSize(10000L) + .build(); + ConnectionSettings connectionSettings = + ConnectionSettings.newBuilder() + .setDefaultDataset(DatasetId.of(DATASET)) + .setNumBufferedRows(10000L) // page size + .setPriority( + QueryJobConfiguration.Priority + .INTERACTIVE) // so that isFastQuerySupported returns false + .setReadClientConnectionConfiguration(clientConnectionConfiguration) + .setUseReadAPI(true) + .build(); + return bigquery.createConnection(connectionSettings); + } + private static Map getNextRow() { rowCnt++; Map row = new HashMap<>(); From e2f3fc8b114edbdd46a07e7463f9fe0099073d4a Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 5 Apr 2022 11:43:31 +0530 Subject: [PATCH 181/277] Added testMultipleRuns --- .../bigquery/it/ITNightlyBigQueryTest.java | 107 ++++++++++++++++-- 1 file changed, 100 insertions(+), 7 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java index e160a21db..33c048366 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java @@ -78,15 +78,18 @@ public class ITNightlyBigQueryTest { // Script will populate NUM_BATCHES*REC_PER_BATCHES number of records (eg: 100*10000 = 1M) private static final int NUM_BATCHES = 35; private static final int REC_PER_BATCHES = 10000; - private static final int LIMIT_RECS = 300000; + private static final int LIMIT_RECS = 300000; // We can plan to read ~ 1M records + private static final int MULTI_LIMIT_RECS = + 300000; // Used for multiquery testcase, a lower limit like 300K should be fine private static int rowCnt = 0; private static BigQuery bigquery; - private static final String QUERY = - String.format( - "select StringField, GeographyField, BooleanField, BigNumericField, IntegerField, NumericField, BytesField, " - + "TimestampField, TimeField, DateField, IntegerArrayField, RecordField.BooleanField, RecordField.StringField ," - + " JSONField, JSONField.hello, JSONField.id from %s.%s order by IntegerField asc LIMIT %s", - DATASET, TABLE, LIMIT_RECS); + private static final String BASE_QUERY = + "select StringField, GeographyField, BooleanField, BigNumericField, IntegerField, NumericField, BytesField, " + + "TimestampField, TimeField, DateField, IntegerArrayField, RecordField.BooleanField, RecordField.StringField ," + + " JSONField, JSONField.hello, JSONField.id from %s.%s order by IntegerField asc LIMIT %s"; + private static final String QUERY = String.format(BASE_QUERY, DATASET, TABLE, LIMIT_RECS); + private static final String MULTI_QUERY = + String.format(BASE_QUERY, DATASET, TABLE, MULTI_LIMIT_RECS); private static final String INVALID_QUERY = String.format( "select into %s.%s order by IntegerField asc LIMIT %s", DATASET, TABLE, LIMIT_RECS); @@ -260,6 +263,96 @@ public void testIterateAndOrder() throws SQLException { connection.cancel(); } + @Test + public void testMultipleRuns() throws SQLException { + + Connection connection = getConnection(); + BigQueryResultSet bigQueryResultSet = connection.executeSelect(MULTI_QUERY); + logger.log(Level.INFO, "Query used: {0}", MULTI_QUERY); + ResultSet rs = bigQueryResultSet.getResultSet(); + int cnt = 0; + int totalCnt = 0; + + int prevIntegerFieldVal = 0; + while (rs.next()) { + if (cnt == 0) { // first row is supposed to be null + assertNull(rs.getString("StringField")); + assertNull(rs.getString("GeographyField")); + Object intAryField = rs.getObject("IntegerArrayField"); + if (intAryField instanceof JsonStringArrayList) { + assertEquals( + new JsonStringArrayList(), + ((JsonStringArrayList) intAryField)); // null array is returned as an empty array + } + assertFalse(rs.getBoolean("BooleanField")); + assertTrue(0.0d == rs.getDouble("BigNumericField")); + assertTrue(0 == rs.getInt("IntegerField")); + assertTrue(0L == rs.getLong("NumericField")); + assertNull(rs.getBytes("BytesField")); + assertNull(rs.getTimestamp("TimestampField")); + assertNull(rs.getTime("TimeField")); + assertNull(rs.getDate("DateField")); + assertNull(rs.getString("JSONField")); + assertFalse(rs.getBoolean("BooleanField_1")); + assertNull(rs.getString("StringField_1")); + assertNull(rs.getString("hello")); // equivalent of testJsonType + assertEquals(0, rs.getInt("id")); + + } else { // remaining rows are supposed to be non null + // check the order of the records + assertTrue(prevIntegerFieldVal < rs.getInt("IntegerField")); + prevIntegerFieldVal = rs.getInt("IntegerField"); + + testForAllDataTypeValues(rs, cnt); // asserts the value of each row + } + ++cnt; + } + connection.cancel(); + totalCnt += cnt; + // Repeat the same run + connection = getConnection(); + bigQueryResultSet = connection.executeSelect(MULTI_QUERY); + rs = bigQueryResultSet.getResultSet(); + cnt = 0; + prevIntegerFieldVal = 0; + while (rs.next()) { + if (cnt == 0) { // first row is supposed to be null + assertNull(rs.getString("StringField")); + assertNull(rs.getString("GeographyField")); + Object intAryField = rs.getObject("IntegerArrayField"); + if (intAryField instanceof JsonStringArrayList) { + assertEquals( + new JsonStringArrayList(), + ((JsonStringArrayList) intAryField)); // null array is returned as an empty array + } + assertFalse(rs.getBoolean("BooleanField")); + assertTrue(0.0d == rs.getDouble("BigNumericField")); + assertTrue(0 == rs.getInt("IntegerField")); + assertTrue(0L == rs.getLong("NumericField")); + assertNull(rs.getBytes("BytesField")); + assertNull(rs.getTimestamp("TimestampField")); + assertNull(rs.getTime("TimeField")); + assertNull(rs.getDate("DateField")); + assertNull(rs.getString("JSONField")); + assertFalse(rs.getBoolean("BooleanField_1")); + assertNull(rs.getString("StringField_1")); + assertNull(rs.getString("hello")); // equivalent of testJsonType + assertEquals(0, rs.getInt("id")); + + } else { // remaining rows are supposed to be non null + // check the order of the records + assertTrue(prevIntegerFieldVal < rs.getInt("IntegerField")); + prevIntegerFieldVal = rs.getInt("IntegerField"); + + testForAllDataTypeValues(rs, cnt); // asserts the value of each row + } + ++cnt; + } + connection.cancel(); + totalCnt += cnt; + assertEquals(MULTI_LIMIT_RECS * 2, totalCnt); + } + // asserts the value of each row private static void testForAllDataTypeValues(ResultSet rs, int cnt) throws SQLException { // Testing JSON type From e8b2705889936d85a9143a3fcb1ded816cc54e6b Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 5 Apr 2022 15:18:23 +0530 Subject: [PATCH 182/277] Added testPositionalParams --- .../bigquery/it/ITNightlyBigQueryTest.java | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java index 33c048366..ef22ee57b 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java @@ -23,6 +23,8 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import com.google.api.services.bigquery.model.QueryParameter; +import com.google.api.services.bigquery.model.QueryParameterType; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryError; import com.google.cloud.bigquery.BigQueryException; @@ -55,6 +57,7 @@ import java.sql.Time; import java.time.LocalTime; import java.time.ZoneId; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -87,6 +90,10 @@ public class ITNightlyBigQueryTest { "select StringField, GeographyField, BooleanField, BigNumericField, IntegerField, NumericField, BytesField, " + "TimestampField, TimeField, DateField, IntegerArrayField, RecordField.BooleanField, RecordField.StringField ," + " JSONField, JSONField.hello, JSONField.id from %s.%s order by IntegerField asc LIMIT %s"; + private static final String POSITIONAL_QUERY = + String.format( + "select StringField, GeographyField, BooleanField, DateField from %s.%s where DateField=? and StringField is NOT NULL LIMIT %s", + DATASET, TABLE, MULTI_LIMIT_RECS); private static final String QUERY = String.format(BASE_QUERY, DATASET, TABLE, LIMIT_RECS); private static final String MULTI_QUERY = String.format(BASE_QUERY, DATASET, TABLE, MULTI_LIMIT_RECS); @@ -353,6 +360,44 @@ public void testMultipleRuns() throws SQLException { assertEquals(MULTI_LIMIT_RECS * 2, totalCnt); } + @Test + public void testPositionalParams() throws SQLException { + Connection connection = getConnection(); + List parameters = new ArrayList<>(); + /* QueryParameter queryParameter = new QueryParameter(); + queryParameter.set("DateField", "2022-01-01"); + parameters.add(queryParameter); + QueryParameterValue val = QueryParameterValue.string("stringValue");*/ + + QueryParameter queryParameterPb = new QueryParameter(); + queryParameterPb.setName("DateField"); + + com.google.api.services.bigquery.model.QueryParameterValue val = + new com.google.api.services.bigquery.model.QueryParameterValue(); + val.set("DateField", "2022-01-01"); + QueryParameterType queryParameterType = + new QueryParameterType().setType(StandardSQLTypeName.DATE.name()); + + queryParameterPb.setParameterValue(val); + queryParameterPb.setParameterType(queryParameterType); + parameters.add(queryParameterPb); + + BigQueryResultSet bigQueryResultSet = + connection.executeSelect(POSITIONAL_QUERY, parameters, null); + logger.log(Level.INFO, "Query used: {0}", POSITIONAL_QUERY); + ResultSet rs = bigQueryResultSet.getResultSet(); + int cnt = 0; + while (rs.next()) { + assertNull(rs.getDate("DateField")); + assertNull(rs.getString("StringField")); + assertNull(rs.getString("GeographyField")); + assertFalse(rs.getBoolean("BooleanField")); + ++cnt; + } + connection.cancel(); + assertEquals(MULTI_LIMIT_RECS, cnt); + } + // asserts the value of each row private static void testForAllDataTypeValues(ResultSet rs, int cnt) throws SQLException { // Testing JSON type From 0efcd967c55fa34c39ca151bd2c97d1ddd9b784e Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 5 Apr 2022 15:29:37 +0530 Subject: [PATCH 183/277] updated positional query --- .../com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java index ef22ee57b..d47e45e9e 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java @@ -92,7 +92,7 @@ public class ITNightlyBigQueryTest { + " JSONField, JSONField.hello, JSONField.id from %s.%s order by IntegerField asc LIMIT %s"; private static final String POSITIONAL_QUERY = String.format( - "select StringField, GeographyField, BooleanField, DateField from %s.%s where DateField=? and StringField is NOT NULL LIMIT %s", + "select StringField, GeographyField, BooleanField, DateField from %s.%s where DateField = ? LIMIT %s", DATASET, TABLE, MULTI_LIMIT_RECS); private static final String QUERY = String.format(BASE_QUERY, DATASET, TABLE, LIMIT_RECS); private static final String MULTI_QUERY = From fb7746891bdd5b3ddb6ddd8fbacd0b76d1d86a00 Mon Sep 17 00:00:00 2001 From: stephwang Date: Wed, 13 Apr 2022 19:36:15 -0400 Subject: [PATCH 184/277] update logic to not use Read API when QueryParameters are passed since it's not supported by the Read API --- .../google/cloud/bigquery/ConnectionImpl.java | 80 ++++++++++--------- .../cloud/bigquery/ConnectionImplTest.java | 36 +++++++-- .../bigquery/it/ITNightlyBigQueryTest.java | 41 ---------- 3 files changed, 72 insertions(+), 85 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 1118d11e1..650285c39 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -148,38 +148,19 @@ public BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException { if (isFastQuerySupported()) { String projectId = bigQueryOptions.getProjectId(); QueryRequest queryRequest = createQueryRequest(connectionSettings, sql, null, null); - return queryRpc(projectId, queryRequest); + return queryRpc(projectId, queryRequest, false); } // use jobs.insert otherwise com.google.api.services.bigquery.model.Job queryJob = createQueryJob(sql, connectionSettings, null, null); JobId jobId = JobId.fromPb(queryJob.getJobReference()); GetQueryResultsResponse firstPage = getQueryResultsFirstPage(jobId); - return getResultSet(firstPage, jobId, sql); + return getResultSet(firstPage, jobId, sql, false); } catch (BigQueryException e) { throw new BigQuerySQLException(e.getMessage(), e, e.getErrors()); } } - @VisibleForTesting - BigQueryResultSet getResultSet(GetQueryResultsResponse firstPage, JobId jobId, String sql) { - if (firstPage.getJobComplete() - && firstPage.getTotalRows() - != null) { // firstPage.getTotalRows() is null if job is not complete - return getSubsequentQueryResultsWithJob( - firstPage.getTotalRows().longValue(), - (long) firstPage.getRows().size(), - jobId, - firstPage); - } else { // job is still running, use dryrun to get Schema - com.google.api.services.bigquery.model.Job dryRunJob = createDryRunJob(sql); - Schema schema = Schema.fromPb(dryRunJob.getStatistics().getQuery().getSchema()); - // TODO: check how can we get totalRows and pageRows while the job is still running. - // `firstPage.getTotalRows()` returns null - return getSubsequentQueryResultsWithJob(null, null, jobId, firstPage, schema); - } - } - /** * This method executes a SQL SELECT query * @@ -205,19 +186,41 @@ public BigQueryResultSet executeSelect( final String projectId = bigQueryOptions.getProjectId(); final QueryRequest queryRequest = createQueryRequest(connectionSettings, sql, parameters, labels); - return queryRpc(projectId, queryRequest); + return queryRpc(projectId, queryRequest, true); } // use jobs.insert otherwise com.google.api.services.bigquery.model.Job queryJob = createQueryJob(sql, connectionSettings, parameters, labels); JobId jobId = JobId.fromPb(queryJob.getJobReference()); GetQueryResultsResponse firstPage = getQueryResultsFirstPage(jobId); - return getResultSet(firstPage, jobId, sql); + return getResultSet(firstPage, jobId, sql, true); } catch (BigQueryException e) { throw new BigQuerySQLException(e.getMessage(), e, e.getErrors()); } } + @VisibleForTesting + BigQueryResultSet getResultSet( + GetQueryResultsResponse firstPage, JobId jobId, String sql, Boolean hasQueryParameters) { + if (firstPage.getJobComplete() + && firstPage.getTotalRows() + != null) { // firstPage.getTotalRows() is null if job is not complete + return getSubsequentQueryResultsWithJob( + firstPage.getTotalRows().longValue(), + (long) firstPage.getRows().size(), + jobId, + firstPage, + hasQueryParameters); + } else { // job is still running, use dryrun to get Schema + com.google.api.services.bigquery.model.Job dryRunJob = createDryRunJob(sql); + Schema schema = Schema.fromPb(dryRunJob.getStatistics().getQuery().getSchema()); + // TODO: check how can we get totalRows and pageRows while the job is still running. + // `firstPage.getTotalRows()` returns null + return getSubsequentQueryResultsWithJob( + null, null, jobId, firstPage, schema, hasQueryParameters); + } + } + static class EndOfFieldValueList extends AbstractList< FieldValue> { // A reference of this class is used as a token to inform the thread @@ -233,7 +236,8 @@ public int size() { } } - private BigQueryResultSet queryRpc(final String projectId, final QueryRequest queryRequest) { + private BigQueryResultSet queryRpc( + final String projectId, final QueryRequest queryRequest, Boolean hasQueryParameters) { com.google.api.services.bigquery.model.QueryResponse results; try { results = @@ -268,7 +272,8 @@ private BigQueryResultSet queryRpc(final String projectId, final QueryRequest qu long pageRows = results.getRows().size(); JobId jobId = JobId.fromPb(results.getJobReference()); GetQueryResultsResponse firstPage = getQueryResultsFirstPage(jobId); - return getSubsequentQueryResultsWithJob(totalRows, pageRows, jobId, firstPage); + return getSubsequentQueryResultsWithJob( + totalRows, pageRows, jobId, firstPage, hasQueryParameters); } } @@ -563,9 +568,13 @@ int getPageCacheSize(Long numBufferedRows, long numRows, Schema schema) { /* Returns query results using either tabledata.list or the high throughput Read API */ @VisibleForTesting BigQueryResultSet getSubsequentQueryResultsWithJob( - Long totalRows, Long pageRows, JobId jobId, GetQueryResultsResponse firstPage) { + Long totalRows, + Long pageRows, + JobId jobId, + GetQueryResultsResponse firstPage, + Boolean hasQueryParameters) { TableId destinationTable = getDestinationTable(jobId); - return useReadAPI(totalRows, pageRows, Schema.fromPb(firstPage.getSchema())) + return useReadAPI(totalRows, pageRows, Schema.fromPb(firstPage.getSchema()), hasQueryParameters) ? highThroughPutRead( destinationTable, firstPage.getTotalRows().longValue(), @@ -583,15 +592,15 @@ BigQueryResultSet getSubsequentQueryResultsWithJob( Long pageRows, JobId jobId, GetQueryResultsResponse firstPage, - Schema schema) { + Schema schema, + Boolean hasQueryParameters) { TableId destinationTable = getDestinationTable(jobId); - return useReadAPI(totalRows, pageRows, schema) + return useReadAPI(totalRows, pageRows, schema, hasQueryParameters) ? highThroughPutRead( destinationTable, totalRows == null ? -1L - : totalRows - .longValue(), // totalRows is null when the job is still running. TODO: Check if + : totalRows, // totalRows is null when the job is still running. TODO: Check if // any workaround is possible schema, getBigQueryResultSetStats( @@ -902,7 +911,7 @@ boolean isFastQuerySupported() { } @VisibleForTesting - boolean useReadAPI(Long totalRows, Long pageRows, Schema schema) { + boolean useReadAPI(Long totalRows, Long pageRows, Schema schema, Boolean hasQueryParameters) { // TODO(prasmish) get this logic review - totalRows and pageRows are returned null when the job // is not complete @@ -915,10 +924,9 @@ boolean useReadAPI(Long totalRows, Long pageRows, Schema schema) { } // Schema schema = Schema.fromPb(tableSchema); - if (containsIntervalType( - schema)) { // finds out if there's an interval type in the schema. Implementation to be used - // until ReadAPI supports Interval - logger.log(Level.INFO, "\n Schema has IntervalType, Disabling ReadAPI"); + // Read API does not yet support Interval Type or QueryParameters + if (containsIntervalType(schema) || hasQueryParameters) { + logger.log(Level.INFO, "\n Schema has IntervalType, or QueryParameters. Disabling ReadAPI"); return false; } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java index 81381ed86..e7794dc13 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java @@ -338,7 +338,11 @@ public void testLegacyQuerySinglePage() throws BigQuerySQLException { doReturn(BQ_RS_MOCK_RES) .when(connectionSpy) .getSubsequentQueryResultsWithJob( - any(Long.class), any(Long.class), any(JobId.class), any(GetQueryResultsResponse.class)); + any(Long.class), + any(Long.class), + any(JobId.class), + any(GetQueryResultsResponse.class), + any(Boolean.class)); when(bigqueryRpcMock.createJobForQuery(any(com.google.api.services.bigquery.model.Job.class))) .thenReturn(jobResponseMock); // RPC call in createQueryJob BigQueryResultSet res = connectionSpy.executeSelect(SQL_QUERY); @@ -442,12 +446,20 @@ public void testExecuteSelectSlow() throws BigQuerySQLException { .getQueryResultsFirstPage(any(JobId.class)); doReturn(BQ_RS_MOCK_RES) .when(connectionSpy) - .getResultSet(any(GetQueryResultsResponse.class), any(JobId.class), any(String.class)); + .getResultSet( + any(GetQueryResultsResponse.class), + any(JobId.class), + any(String.class), + any(Boolean.class)); BigQueryResultSet res = connectionSpy.executeSelect(SQL_QUERY); assertEquals(res.getTotalRows(), 2); assertEquals(QUERY_SCHEMA, res.getSchema()); verify(connectionSpy, times(1)) - .getResultSet(any(GetQueryResultsResponse.class), any(JobId.class), any(String.class)); + .getResultSet( + any(GetQueryResultsResponse.class), + any(JobId.class), + any(String.class), + any(Boolean.class)); } @Test @@ -473,12 +485,20 @@ public void testExecuteSelectSlowWithParams() throws BigQuerySQLException { .getQueryResultsFirstPage(any(JobId.class)); doReturn(BQ_RS_MOCK_RES) .when(connectionSpy) - .getResultSet(any(GetQueryResultsResponse.class), any(JobId.class), any(String.class)); + .getResultSet( + any(GetQueryResultsResponse.class), + any(JobId.class), + any(String.class), + any(Boolean.class)); BigQueryResultSet res = connectionSpy.executeSelect(SQL_QUERY, parameters, labels); assertEquals(res.getTotalRows(), 2); assertEquals(QUERY_SCHEMA, res.getSchema()); verify(connectionSpy, times(1)) - .getResultSet(any(GetQueryResultsResponse.class), any(JobId.class), any(String.class)); + .getResultSet( + any(GetQueryResultsResponse.class), + any(JobId.class), + any(String.class), + any(Boolean.class)); } @Test @@ -488,7 +508,7 @@ public void testGetSubsequentQueryResultsWithJob() { BigQueryResultSetStats bqRsStats = mock(BigQueryResultSetStats.class); doReturn(true) .when(connectionSpy) - .useReadAPI(any(Long.class), any(Long.class), any(Schema.class)); + .useReadAPI(any(Long.class), any(Long.class), any(Schema.class), any(Boolean.class)); doReturn(BQ_RS_MOCK_RES) .when(connectionSpy) .highThroughPutRead( @@ -501,11 +521,11 @@ public void testGetSubsequentQueryResultsWithJob() { doReturn(bqRsStats).when(connectionSpy).getBigQueryResultSetStats(any(JobId.class)); BigQueryResultSet res = connectionSpy.getSubsequentQueryResultsWithJob( - 10000L, 100L, jobId, GET_QUERY_RESULTS_RESPONSE); + 10000L, 100L, jobId, GET_QUERY_RESULTS_RESPONSE, false); assertEquals(res.getTotalRows(), 2); assertEquals(QUERY_SCHEMA, res.getSchema()); verify(connectionSpy, times(1)) - .getSubsequentQueryResultsWithJob(10000L, 100L, jobId, GET_QUERY_RESULTS_RESPONSE); + .getSubsequentQueryResultsWithJob(10000L, 100L, jobId, GET_QUERY_RESULTS_RESPONSE, false); } @Test diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java index d47e45e9e..e5ff18826 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java @@ -23,8 +23,6 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import com.google.api.services.bigquery.model.QueryParameter; -import com.google.api.services.bigquery.model.QueryParameterType; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryError; import com.google.cloud.bigquery.BigQueryException; @@ -57,7 +55,6 @@ import java.sql.Time; import java.time.LocalTime; import java.time.ZoneId; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -360,44 +357,6 @@ public void testMultipleRuns() throws SQLException { assertEquals(MULTI_LIMIT_RECS * 2, totalCnt); } - @Test - public void testPositionalParams() throws SQLException { - Connection connection = getConnection(); - List parameters = new ArrayList<>(); - /* QueryParameter queryParameter = new QueryParameter(); - queryParameter.set("DateField", "2022-01-01"); - parameters.add(queryParameter); - QueryParameterValue val = QueryParameterValue.string("stringValue");*/ - - QueryParameter queryParameterPb = new QueryParameter(); - queryParameterPb.setName("DateField"); - - com.google.api.services.bigquery.model.QueryParameterValue val = - new com.google.api.services.bigquery.model.QueryParameterValue(); - val.set("DateField", "2022-01-01"); - QueryParameterType queryParameterType = - new QueryParameterType().setType(StandardSQLTypeName.DATE.name()); - - queryParameterPb.setParameterValue(val); - queryParameterPb.setParameterType(queryParameterType); - parameters.add(queryParameterPb); - - BigQueryResultSet bigQueryResultSet = - connection.executeSelect(POSITIONAL_QUERY, parameters, null); - logger.log(Level.INFO, "Query used: {0}", POSITIONAL_QUERY); - ResultSet rs = bigQueryResultSet.getResultSet(); - int cnt = 0; - while (rs.next()) { - assertNull(rs.getDate("DateField")); - assertNull(rs.getString("StringField")); - assertNull(rs.getString("GeographyField")); - assertFalse(rs.getBoolean("BooleanField")); - ++cnt; - } - connection.cancel(); - assertEquals(MULTI_LIMIT_RECS, cnt); - } - // asserts the value of each row private static void testForAllDataTypeValues(ResultSet rs, int cnt) throws SQLException { // Testing JSON type From 948621795578c13e5de1e24e1f627d49246be25d Mon Sep 17 00:00:00 2001 From: stephwang Date: Mon, 18 Apr 2022 17:46:11 -0400 Subject: [PATCH 185/277] update support for query parameters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit deps: update dependency com.google.oauth-client:google-oauth-client-jetty to v1.33.3 (#1978) [![WhiteSource Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com) This PR contains the following updates: | Package | Change | Age | Adoption | Passing | Confidence | |---|---|---|---|---|---| | [com.google.oauth-client:google-oauth-client-jetty](https://togithub.com/googleapis/google-oauth-java-client) | `1.33.2` -> `1.33.3` | [![age](https://badges.renovateapi.com/packages/maven/com.google.oauth-client:google-oauth-client-jetty/1.33.3/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/maven/com.google.oauth-client:google-oauth-client-jetty/1.33.3/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/maven/com.google.oauth-client:google-oauth-client-jetty/1.33.3/compatibility-slim/1.33.2)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/maven/com.google.oauth-client:google-oauth-client-jetty/1.33.3/confidence-slim/1.33.2)](https://docs.renovatebot.com/merge-confidence/) | ---

    googleapis/google-oauth-java-client [Compare Source](https://togithub.com/googleapis/google-oauth-java-client/compare/v1.33.2...v1.33.3)
    --- 📅 **Schedule**: At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, click this checkbox. --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#github/googleapis/java-bigquery). deps: update dependency com.google.oauth-client:google-oauth-client-java6 to v1.33.3 (#1977) [![WhiteSource Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com) This PR contains the following updates: | Package | Change | Age | Adoption | Passing | Confidence | |---|---|---|---|---|---| | [com.google.oauth-client:google-oauth-client-java6](https://togithub.com/googleapis/google-oauth-java-client) | `1.33.2` -> `1.33.3` | [![age](https://badges.renovateapi.com/packages/maven/com.google.oauth-client:google-oauth-client-java6/1.33.3/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/maven/com.google.oauth-client:google-oauth-client-java6/1.33.3/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/maven/com.google.oauth-client:google-oauth-client-java6/1.33.3/compatibility-slim/1.33.2)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/maven/com.google.oauth-client:google-oauth-client-java6/1.33.3/confidence-slim/1.33.2)](https://docs.renovatebot.com/merge-confidence/) | ---
    googleapis/google-oauth-java-client [Compare Source](https://togithub.com/googleapis/google-oauth-java-client/compare/v1.33.2...v1.33.3)
    --- 📅 **Schedule**: At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, click this checkbox. --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#github/googleapis/java-bigquery). --- .../com/google/cloud/bigquery/Connection.java | 3 +- .../google/cloud/bigquery/ConnectionImpl.java | 56 +++++++++++++-- .../com/google/cloud/bigquery/Parameter.java | 70 +++++++++++++++++++ .../cloud/bigquery/ConnectionImplTest.java | 2 +- .../cloud/bigquery/it/ITBigQueryTest.java | 55 ++++++++++++++- samples/install-without-bom/pom.xml | 4 +- samples/snapshot/pom.xml | 4 +- samples/snippets/pom.xml | 4 +- 8 files changed, 180 insertions(+), 18 deletions(-) create mode 100644 google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Parameter.java diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java index 18b32f363..bf0740ae9 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java @@ -17,7 +17,6 @@ package com.google.cloud.bigquery; import com.google.api.core.BetaApi; -import com.google.api.services.bigquery.model.QueryParameter; import java.util.List; import java.util.Map; @@ -92,6 +91,6 @@ public interface Connection { */ @BetaApi BigQueryResultSet executeSelect( - String sql, List parameters, Map labels) + String sql, List parameters, Map labels) throws BigQuerySQLException; } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 650285c39..789fe3ea4 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -42,6 +42,7 @@ import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; import java.io.IOException; import java.util.AbstractList; @@ -178,7 +179,7 @@ public BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException { @BetaApi @Override public BigQueryResultSet executeSelect( - String sql, List parameters, Map labels) + String sql, List parameters, Map labels) throws BigQuerySQLException { try { // use jobs.query if possible @@ -186,14 +187,14 @@ public BigQueryResultSet executeSelect( final String projectId = bigQueryOptions.getProjectId(); final QueryRequest queryRequest = createQueryRequest(connectionSettings, sql, parameters, labels); - return queryRpc(projectId, queryRequest, true); + return queryRpc(projectId, queryRequest, parameters != null); } // use jobs.insert otherwise com.google.api.services.bigquery.model.Job queryJob = createQueryJob(sql, connectionSettings, parameters, labels); JobId jobId = JobId.fromPb(queryJob.getJobReference()); GetQueryResultsResponse firstPage = getQueryResultsFirstPage(jobId); - return getResultSet(firstPage, jobId, sql, true); + return getResultSet(firstPage, jobId, sql, parameters != null); } catch (BigQueryException e) { throw new BigQuerySQLException(e.getMessage(), e, e.getErrors()); } @@ -967,7 +968,7 @@ private boolean containsIntervalType(Schema schema) { QueryRequest createQueryRequest( ConnectionSettings connectionSettings, String sql, - List queryParameters, + List queryParameters, Map labels) { QueryRequest content = new QueryRequest(); String requestId = UUID.randomUUID().toString(); @@ -988,7 +989,21 @@ QueryRequest createQueryRequest( content.setMaxResults(connectionSettings.getMaxResults()); } if (queryParameters != null) { - content.setQueryParameters(queryParameters); + // content.setQueryParameters(queryParameters); + if (queryParameters.get(0).getName() == null) { + // If query parameter name is unset, then assume mode is positional + content.setParameterMode("POSITIONAL"); + // pass query parameters + List queryParametersPb = + Lists.transform(queryParameters, POSITIONAL_PARAMETER_TO_PB_FUNCTION); + content.setQueryParameters(queryParametersPb); + } else { + content.setParameterMode("NAMED"); + // pass query parameters + List queryParametersPb = + Lists.transform(queryParameters, NAMED_PARAMETER_TO_PB_FUNCTION); + content.setQueryParameters(queryParametersPb); + } } if (connectionSettings.getCreateSession() != null) { content.setCreateSession(connectionSettings.getCreateSession()); @@ -1008,18 +1023,26 @@ QueryRequest createQueryRequest( com.google.api.services.bigquery.model.Job createQueryJob( String sql, ConnectionSettings connectionSettings, - List queryParameters, + List queryParameters, Map labels) { com.google.api.services.bigquery.model.JobConfiguration configurationPb = new com.google.api.services.bigquery.model.JobConfiguration(); JobConfigurationQuery queryConfigurationPb = new JobConfigurationQuery(); queryConfigurationPb.setQuery(sql); if (queryParameters != null) { - queryConfigurationPb.setQueryParameters(queryParameters); if (queryParameters.get(0).getName() == null) { + // If query parameter name is unset, then assume mode is positional queryConfigurationPb.setParameterMode("POSITIONAL"); + // pass query parameters + List queryParametersPb = + Lists.transform(queryParameters, POSITIONAL_PARAMETER_TO_PB_FUNCTION); + queryConfigurationPb.setQueryParameters(queryParametersPb); } else { queryConfigurationPb.setParameterMode("NAMED"); + // pass query parameters + List queryParametersPb = + Lists.transform(queryParameters, NAMED_PARAMETER_TO_PB_FUNCTION); + queryConfigurationPb.setQueryParameters(queryParametersPb); } } if (connectionSettings.getDestinationTable() != null) { @@ -1155,4 +1178,23 @@ private com.google.api.services.bigquery.model.Job createDryRunJob(String sql) { } return dryRunJob; } + + // Convert from Parameter wrapper class to positional QueryParameter generated class + private static final Function POSITIONAL_PARAMETER_TO_PB_FUNCTION = + value -> { + QueryParameter queryParameterPb = new QueryParameter(); + queryParameterPb.setParameterValue(value.getQueryParameterValue().toValuePb()); + queryParameterPb.setParameterType(value.getQueryParameterValue().toTypePb()); + return queryParameterPb; + }; + + // Convert from Parameter wrapper class to name QueryParameter generated class + private static final Function NAMED_PARAMETER_TO_PB_FUNCTION = + value -> { + QueryParameter queryParameterPb = new QueryParameter(); + queryParameterPb.setName(value.getName()); + queryParameterPb.setParameterValue(value.getQueryParameterValue().toValuePb()); + queryParameterPb.setParameterType(value.getQueryParameterValue().toTypePb()); + return queryParameterPb; + }; } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Parameter.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Parameter.java new file mode 100644 index 000000000..0f0796eb9 --- /dev/null +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Parameter.java @@ -0,0 +1,70 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import com.google.auto.value.AutoValue; +import javax.annotation.Nullable; + +/* Wrapper class for query parameters */ +@AutoValue +public abstract class Parameter { + Parameter() { + // Package private so users can't subclass it but AutoValue can. + } + + /** + * Returns the name of the query parameter. If unset, this is a positional parameter. Otherwise, + * should be unique within a query. + * + * @return value or {@code null} for none + */ + @Nullable + public abstract String getName(); + + /** Returns the value for a query parameter along with its type. */ + public abstract QueryParameterValue getQueryParameterValue(); + + /** Returns a builder pre-populated using the current values of this field. */ + public abstract Builder toBuilder(); + + /** Returns a builder for a {@code Parameter} object. */ + public static Builder newBuilder() { + return new AutoValue_Parameter.Builder(); + } + + @AutoValue.Builder + public abstract static class Builder { + + /** + * [Optional] Sets the name of the query parameter. If unset, this is a positional parameter. + * Otherwise, should be unique within a query. + * + * @param name name or {@code null} for none + */ + public abstract Builder setName(String name); + + /** + * Sets the the value for a query parameter along with its type. + * + * @param parameter parameter or {@code null} for none + */ + public abstract Builder setQueryParameterValue(QueryParameterValue parameter); + + /** Creates a {@code Parameter} object. */ + public abstract Parameter build(); + } +} diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java index e7794dc13..06620d8d2 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java @@ -465,7 +465,7 @@ public void testExecuteSelectSlow() throws BigQuerySQLException { @Test public void testExecuteSelectSlowWithParams() throws BigQuerySQLException { ConnectionImpl connectionSpy = Mockito.spy(connection); - List parameters = new ArrayList<>(); + List parameters = new ArrayList<>(); Map labels = new HashMap<>(); doReturn(false).when(connectionSpy).isFastQuerySupported(); com.google.api.services.bigquery.model.JobStatistics jobStatistics = diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 5e352b8bd..23e27bac3 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -87,6 +87,7 @@ import com.google.cloud.bigquery.Model; import com.google.cloud.bigquery.ModelId; import com.google.cloud.bigquery.ModelInfo; +import com.google.cloud.bigquery.Parameter; import com.google.cloud.bigquery.ParquetOptions; import com.google.cloud.bigquery.PolicyTags; import com.google.cloud.bigquery.QueryJobConfiguration; @@ -3285,7 +3286,6 @@ public void testQueryParameterModeWithDryRun() { assertNotNull(statistics.getTotalBytesProcessed()); } - /* TODO(prasmish): replicate relevant parts of the test case for executeSelect */ @Test public void testPositionalQueryParameters() throws InterruptedException { String query = @@ -3356,7 +3356,29 @@ public void testPositionalQueryParameters() throws InterruptedException { } } - /* TODO(prasmish): replicate relevant parts of the test case for executeSelect */ + /* TODO(prasmish): expand below test case with all the fields shown in the above test case */ + @Test + public void testExecuteSelectWithPositionalQueryParameters() throws BigQuerySQLException { + String query = + "SELECT TimestampField, StringField FROM " + + TABLE_ID.getTable() + + " WHERE StringField = ?" + + " AND TimestampField > ?"; + QueryParameterValue stringParameter = QueryParameterValue.string("stringValue"); + QueryParameterValue timestampParameter = + QueryParameterValue.timestamp("2014-01-01 07:00:00.000000+00:00"); + Parameter stringParam = Parameter.newBuilder().setQueryParameterValue(stringParameter).build(); + Parameter timeStampParam = + Parameter.newBuilder().setQueryParameterValue(timestampParameter).build(); + + ConnectionSettings connectionSettings = + ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); + Connection connection = bigquery.createConnection(connectionSettings); + List parameters = ImmutableList.of(stringParam, timeStampParam); + BigQueryResultSet rs = connection.executeSelect(query, parameters, null); + assertEquals(2, rs.getTotalRows()); + } + @Test public void testNamedQueryParameters() throws InterruptedException { String query = @@ -3379,6 +3401,35 @@ public void testNamedQueryParameters() throws InterruptedException { assertEquals(2, Iterables.size(result.getValues())); } + @Test + public void testExecuteSelectWithNamedQueryParameters() throws BigQuerySQLException { + String query = + "SELECT TimestampField, StringField, BooleanField FROM " + + TABLE_ID.getTable() + + " WHERE StringField = @stringParam" + + " AND IntegerField IN UNNEST(@integerList)"; + QueryParameterValue stringParameter = QueryParameterValue.string("stringValue"); + QueryParameterValue intArrayParameter = + QueryParameterValue.array(new Integer[] {3, 4}, Integer.class); + Parameter stringParam = + Parameter.newBuilder() + .setName("stringParam") + .setQueryParameterValue(stringParameter) + .build(); + Parameter intArrayParam = + Parameter.newBuilder() + .setName("integerList") + .setQueryParameterValue(intArrayParameter) + .build(); + + ConnectionSettings connectionSettings = + ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); + Connection connection = bigquery.createConnection(connectionSettings); + List parameters = ImmutableList.of(stringParam, intArrayParam); + BigQueryResultSet rs = connection.executeSelect(query, parameters, null); + assertEquals(2, rs.getTotalRows()); + } + /* TODO(prasmish): replicate relevant parts of the test case for executeSelect */ @Test public void testStructNamedQueryParameters() throws InterruptedException { diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index 62c6ecea3..95115402c 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -52,12 +52,12 @@ com.google.oauth-client google-oauth-client-java6 - 1.33.2 + 1.33.3 com.google.oauth-client google-oauth-client-jetty - 1.33.2 + 1.33.3 diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 179033a52..7bda0e2da 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -50,12 +50,12 @@ com.google.oauth-client google-oauth-client-java6 - 1.33.2 + 1.33.3 com.google.oauth-client google-oauth-client-jetty - 1.33.2 + 1.33.3 diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index 072ac96fe..d75d36445 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -66,12 +66,12 @@ com.google.oauth-client google-oauth-client-java6 - 1.33.2 + 1.33.3 com.google.oauth-client google-oauth-client-jetty - 1.33.2 + 1.33.3 From dcbd33ed0adb1161fd21f1bfca71ba9364311a42 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 14 Apr 2022 13:26:36 +0000 Subject: [PATCH 186/277] chore(main): release 2.10.8 (#1979) :robot: I have created a release *beep* *boop* --- ### [2.10.8](https://github.com/googleapis/java-bigquery/compare/v2.10.7...v2.10.8) (2022-04-14) ### Dependencies * update dependency com.google.oauth-client:google-oauth-client-java6 to v1.33.3 ([#1977](https://github.com/googleapis/java-bigquery/issues/1977)) ([b084791](https://github.com/googleapis/java-bigquery/commit/b08479180e212bccae166f3a675d46fb658d3ce8)) * update dependency com.google.oauth-client:google-oauth-client-jetty to v1.33.3 ([#1978](https://github.com/googleapis/java-bigquery/issues/1978)) ([a24ce6e](https://github.com/googleapis/java-bigquery/commit/a24ce6ead65f3fceff6573e24c10c324650d907b)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- CHANGELOG.md | 8 ++++++++ benchmark/pom.xml | 2 +- google-cloud-bigquery/pom.xml | 4 ++-- pom.xml | 4 ++-- samples/snapshot/pom.xml | 2 +- versions.txt | 2 +- 6 files changed, 15 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15e00891f..499427191 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +### [2.10.8](https://github.com/googleapis/java-bigquery/compare/v2.10.7...v2.10.8) (2022-04-14) + + +### Dependencies + +* update dependency com.google.oauth-client:google-oauth-client-java6 to v1.33.3 ([#1977](https://github.com/googleapis/java-bigquery/issues/1977)) ([b084791](https://github.com/googleapis/java-bigquery/commit/b08479180e212bccae166f3a675d46fb658d3ce8)) +* update dependency com.google.oauth-client:google-oauth-client-jetty to v1.33.3 ([#1978](https://github.com/googleapis/java-bigquery/issues/1978)) ([a24ce6e](https://github.com/googleapis/java-bigquery/commit/a24ce6ead65f3fceff6573e24c10c324650d907b)) + ### [2.10.7](https://github.com/googleapis/java-bigquery/compare/v2.10.6...v2.10.7) (2022-04-08) diff --git a/benchmark/pom.xml b/benchmark/pom.xml index 0b59ff743..d8722a1e6 100644 --- a/benchmark/pom.xml +++ b/benchmark/pom.xml @@ -6,7 +6,7 @@ google-cloud-bigquery-parent com.google.cloud - 2.10.8-SNAPSHOT + 2.10.8 diff --git a/google-cloud-bigquery/pom.xml b/google-cloud-bigquery/pom.xml index 50b7ec096..718fc6174 100644 --- a/google-cloud-bigquery/pom.xml +++ b/google-cloud-bigquery/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-bigquery - 2.10.8-SNAPSHOT + 2.10.8 jar BigQuery https://github.com/googleapis/java-bigquery @@ -11,7 +11,7 @@ com.google.cloud google-cloud-bigquery-parent - 2.10.8-SNAPSHOT + 2.10.8 google-cloud-bigquery diff --git a/pom.xml b/pom.xml index 0b1e513e1..b6dddce9e 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-bigquery-parent pom - 2.10.8-SNAPSHOT + 2.10.8 BigQuery Parent https://github.com/googleapis/java-bigquery @@ -99,7 +99,7 @@ com.google.cloud google-cloud-bigquery - 2.10.8-SNAPSHOT + 2.10.8 diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 7bda0e2da..4104c783d 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -44,7 +44,7 @@ com.google.cloud google-cloud-bigquery - 2.10.8-SNAPSHOT + 2.10.8 diff --git a/versions.txt b/versions.txt index 03fada45b..1490c8054 100644 --- a/versions.txt +++ b/versions.txt @@ -1,4 +1,4 @@ # Format: # module:released-version:current-version -google-cloud-bigquery:2.10.7:2.10.8-SNAPSHOT \ No newline at end of file +google-cloud-bigquery:2.10.8:2.10.8 \ No newline at end of file From 5a20d420bead374913023a3a81c4cfa432db08f8 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 14 Apr 2022 18:50:26 -0400 Subject: [PATCH 187/277] chore(main): release 2.10.9-SNAPSHOT (#1980) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- benchmark/pom.xml | 2 +- google-cloud-bigquery/pom.xml | 4 ++-- pom.xml | 4 ++-- samples/snapshot/pom.xml | 2 +- versions.txt | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/benchmark/pom.xml b/benchmark/pom.xml index d8722a1e6..e9466fcb9 100644 --- a/benchmark/pom.xml +++ b/benchmark/pom.xml @@ -6,7 +6,7 @@ google-cloud-bigquery-parent com.google.cloud - 2.10.8 + 2.10.9-SNAPSHOT diff --git a/google-cloud-bigquery/pom.xml b/google-cloud-bigquery/pom.xml index 718fc6174..81fc9437f 100644 --- a/google-cloud-bigquery/pom.xml +++ b/google-cloud-bigquery/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-bigquery - 2.10.8 + 2.10.9-SNAPSHOT jar BigQuery https://github.com/googleapis/java-bigquery @@ -11,7 +11,7 @@ com.google.cloud google-cloud-bigquery-parent - 2.10.8 + 2.10.9-SNAPSHOT google-cloud-bigquery diff --git a/pom.xml b/pom.xml index b6dddce9e..5cd38742c 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-bigquery-parent pom - 2.10.8 + 2.10.9-SNAPSHOT BigQuery Parent https://github.com/googleapis/java-bigquery @@ -99,7 +99,7 @@ com.google.cloud google-cloud-bigquery - 2.10.8 + 2.10.9-SNAPSHOT diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 4104c783d..eb3aa78bd 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -44,7 +44,7 @@ com.google.cloud google-cloud-bigquery - 2.10.8 + 2.10.9-SNAPSHOT diff --git a/versions.txt b/versions.txt index 1490c8054..b771ef888 100644 --- a/versions.txt +++ b/versions.txt @@ -1,4 +1,4 @@ # Format: # module:released-version:current-version -google-cloud-bigquery:2.10.8:2.10.8 \ No newline at end of file +google-cloud-bigquery:2.10.8:2.10.9-SNAPSHOT \ No newline at end of file From 384dece13637c3f00ec1ec75f4b80e88113d71ea Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 15 Apr 2022 21:32:27 +0200 Subject: [PATCH 188/277] deps: update dependency com.google.cloud:native-image-support to v0.13.1 (#1982) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [![WhiteSource Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com) This PR contains the following updates: | Package | Change | Age | Adoption | Passing | Confidence | |---|---|---|---|---|---| | [com.google.cloud:native-image-support](https://togithub.com/googleapis/java-core) | `0.12.11` -> `0.13.1` | [![age](https://badges.renovateapi.com/packages/maven/com.google.cloud:native-image-support/0.13.1/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/maven/com.google.cloud:native-image-support/0.13.1/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/maven/com.google.cloud:native-image-support/0.13.1/compatibility-slim/0.12.11)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/maven/com.google.cloud:native-image-support/0.13.1/confidence-slim/0.12.11)](https://docs.renovatebot.com/merge-confidence/) | --- ### Configuration 📅 **Schedule**: At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Renovate will not automatically rebase this PR, because other commits have been found. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, click this checkbox. --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#github/googleapis/java-bigquery). --- README.md | 4 ++-- samples/native-image-sample/pom.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 997add9ff..6b8f45c20 100644 --- a/README.md +++ b/README.md @@ -59,13 +59,13 @@ implementation 'com.google.cloud:google-cloud-bigquery' If you are using Gradle without BOM, add this to your dependencies ```Groovy -implementation 'com.google.cloud:google-cloud-bigquery:2.10.7' +implementation 'com.google.cloud:google-cloud-bigquery:2.10.8' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.10.7" +libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.10.8" ``` ## Authentication diff --git a/samples/native-image-sample/pom.xml b/samples/native-image-sample/pom.xml index f4893182a..4d22a0c99 100644 --- a/samples/native-image-sample/pom.xml +++ b/samples/native-image-sample/pom.xml @@ -93,7 +93,7 @@ com.google.cloud native-image-support - 0.12.11 + 0.13.1 org.junit.vintage From 1f22d4fa4a09552d2b0385dbc29f1abaf74b1e47 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 15 Apr 2022 21:40:28 +0200 Subject: [PATCH 189/277] chore(deps): update dependency com.google.cloud:google-cloud-bigquery to v2.10.8 (#1981) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [![WhiteSource Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com) This PR contains the following updates: | Package | Change | Age | Adoption | Passing | Confidence | |---|---|---|---|---|---| | [com.google.cloud:google-cloud-bigquery](https://togithub.com/googleapis/java-bigquery) | `2.10.7` -> `2.10.8` | [![age](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-bigquery/2.10.8/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-bigquery/2.10.8/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-bigquery/2.10.8/compatibility-slim/2.10.7)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-bigquery/2.10.8/confidence-slim/2.10.7)](https://docs.renovatebot.com/merge-confidence/) | --- ### Release Notes
    googleapis/java-bigquery ### [`v2.10.8`](https://togithub.com/googleapis/java-bigquery/blob/HEAD/CHANGELOG.md#​2108-httpsgithubcomgoogleapisjava-bigquerycomparev2107v2108-2022-04-14) [Compare Source](https://togithub.com/googleapis/java-bigquery/compare/v2.10.7...v2.10.8)
    --- ### Configuration 📅 **Schedule**: At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Renovate will not automatically rebase this PR, because other commits have been found. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, click this checkbox. --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#github/googleapis/java-bigquery). --- README.md | 2 +- samples/install-without-bom/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6b8f45c20..f0e0c1053 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ If you are using Maven without BOM, add this to your dependencies: com.google.cloud google-cloud-bigquery - 2.10.7 + 2.10.8 ``` diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index 95115402c..5e342ab79 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -45,7 +45,7 @@ com.google.cloud google-cloud-bigquery - 2.10.7 + 2.10.8 From 7f18b6abf9b81c46ab280c2a5ea806b5adfaff26 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 15 Apr 2022 21:44:30 +0200 Subject: [PATCH 190/277] deps: update dependency com.google.cloud:google-cloud-shared-dependencies to v2.10.0 (#1983) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [![WhiteSource Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com) This PR contains the following updates: | Package | Change | Age | Adoption | Passing | Confidence | |---|---|---|---|---|---| | [com.google.cloud:google-cloud-shared-dependencies](https://togithub.com/googleapis/java-shared-dependencies) | `2.9.0` -> `2.10.0` | [![age](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-shared-dependencies/2.10.0/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-shared-dependencies/2.10.0/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-shared-dependencies/2.10.0/compatibility-slim/2.9.0)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-shared-dependencies/2.10.0/confidence-slim/2.9.0)](https://docs.renovatebot.com/merge-confidence/) | --- ### Release Notes
    googleapis/java-shared-dependencies ### [`v2.10.0`](https://togithub.com/googleapis/java-shared-dependencies/blob/HEAD/CHANGELOG.md#​2100-httpsgithubcomgoogleapisjava-shared-dependenciescomparev290v2100-2022-04-15) [Compare Source](https://togithub.com/googleapis/java-shared-dependencies/compare/v2.9.0...v2.10.0) ##### Dependencies - google-cloud-core 2.6.0 ([#​668](https://togithub.com/googleapis/java-shared-dependencies/issues/668)) ([22f4d18](https://togithub.com/googleapis/java-shared-dependencies/commit/22f4d1809cbb9848174b3569ab527e4bef00d443)) - reverting protobuf to 3.19.4 ([#​657](https://togithub.com/googleapis/java-shared-dependencies/issues/657)) ([8501e6d](https://togithub.com/googleapis/java-shared-dependencies/commit/8501e6d842c10d2370bbd5d5246070134336bddd)) - update dependency com.fasterxml.jackson:jackson-bom to v2.13.2.20220328 ([#​646](https://togithub.com/googleapis/java-shared-dependencies/issues/646)) ([7bfd6d7](https://togithub.com/googleapis/java-shared-dependencies/commit/7bfd6d7073859d1955b91b368c6713a72ffa14b6)) - update dependency com.google.api-client:google-api-client-bom to v1.34.0 ([#​662](https://togithub.com/googleapis/java-shared-dependencies/issues/662)) ([1b8e378](https://togithub.com/googleapis/java-shared-dependencies/commit/1b8e378fe0ccf2a28c759868caaf5ba593a95728)) - update dependency com.google.errorprone:error_prone_annotations to v2.12.1 ([#​652](https://togithub.com/googleapis/java-shared-dependencies/issues/652)) ([1cc80ee](https://togithub.com/googleapis/java-shared-dependencies/commit/1cc80ee984ebcad9bc2a95e2f28c0a49fe924b37)) - update dependency com.google.errorprone:error_prone_annotations to v2.13.0 ([#​669](https://togithub.com/googleapis/java-shared-dependencies/issues/669)) ([61b7834](https://togithub.com/googleapis/java-shared-dependencies/commit/61b78341b34a251722be4805a6bdd895cd64836c)) - update dependency com.google.http-client:google-http-client-bom to v1.41.6 ([#​654](https://togithub.com/googleapis/java-shared-dependencies/issues/654)) ([140ef40](https://togithub.com/googleapis/java-shared-dependencies/commit/140ef405bc17ed83f5ce177df59affca14fbe49c)) - update dependency com.google.http-client:google-http-client-bom to v1.41.7 ([#​658](https://togithub.com/googleapis/java-shared-dependencies/issues/658)) ([f6f93e5](https://togithub.com/googleapis/java-shared-dependencies/commit/f6f93e5b9172c9684623c4c148e4a8fe2fae1e94)) - update dependency com.google.oauth-client:google-oauth-client-bom to v1.33.2 ([#​655](https://togithub.com/googleapis/java-shared-dependencies/issues/655)) ([20cd9ed](https://togithub.com/googleapis/java-shared-dependencies/commit/20cd9eda112c96836a5ab7485a4247ed2bc0edb8)) - update dependency com.google.oauth-client:google-oauth-client-bom to v1.33.3 ([#​663](https://togithub.com/googleapis/java-shared-dependencies/issues/663)) ([f011a46](https://togithub.com/googleapis/java-shared-dependencies/commit/f011a46c551dba16851b4f8c919c40452fc5d5c3)) - update dependency com.google.protobuf:protobuf-bom to v3.20.0 ([#​651](https://togithub.com/googleapis/java-shared-dependencies/issues/651)) ([ad2ff73](https://togithub.com/googleapis/java-shared-dependencies/commit/ad2ff73207dd6493321c77d8eca0022baf13b4ce)) - update dependency io.grpc:grpc-bom to v1.45.1 ([#​647](https://togithub.com/googleapis/java-shared-dependencies/issues/647)) ([38e46fc](https://togithub.com/googleapis/java-shared-dependencies/commit/38e46fc4f03af0a02f30ce4a2fa222c71797ae15)) - update dependency org.checkerframework:checker-qual to v3.21.4 ([#​650](https://togithub.com/googleapis/java-shared-dependencies/issues/650)) ([125e80a](https://togithub.com/googleapis/java-shared-dependencies/commit/125e80ab2c3225a00c03f5ff5de94872ebb94303)) - update gax.version to v2.15.0 ([#​649](https://togithub.com/googleapis/java-shared-dependencies/issues/649)) ([c7f32a6](https://togithub.com/googleapis/java-shared-dependencies/commit/c7f32a68b14520104432282ac9598643700162eb)) - update gax.version to v2.16.0 ([#​664](https://togithub.com/googleapis/java-shared-dependencies/issues/664)) ([caaf941](https://togithub.com/googleapis/java-shared-dependencies/commit/caaf941643af04295f5527a0144587d7bf040862)) - update google.common-protos.version to v2.8.1 ([#​656](https://togithub.com/googleapis/java-shared-dependencies/issues/656)) ([df4a4a2](https://togithub.com/googleapis/java-shared-dependencies/commit/df4a4a2130a3cdb2965ea42962d1ea6a85506ba7)) - update google.common-protos.version to v2.8.2 ([#​659](https://togithub.com/googleapis/java-shared-dependencies/issues/659)) ([b499e2b](https://togithub.com/googleapis/java-shared-dependencies/commit/b499e2bc99506d48d26e35bf6e68c09409ce8b11)) - update google.common-protos.version to v2.8.3 ([#​660](https://togithub.com/googleapis/java-shared-dependencies/issues/660)) ([461081c](https://togithub.com/googleapis/java-shared-dependencies/commit/461081c0cf73057c1f6e07fc573453ad467a60ae)) - update iam.version to v1.3.0 ([#​648](https://togithub.com/googleapis/java-shared-dependencies/issues/648)) ([6670c4f](https://togithub.com/googleapis/java-shared-dependencies/commit/6670c4f61fcf075c543bfd148eea49796e0662ce)) - update iam.version to v1.3.1 ([#​661](https://togithub.com/googleapis/java-shared-dependencies/issues/661)) ([cc8fbe6](https://togithub.com/googleapis/java-shared-dependencies/commit/cc8fbe6eae03341c2ece7d3356febc843a74a812))
    --- ### Configuration 📅 **Schedule**: At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Renovate will not automatically rebase this PR, because other commits have been found. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, click this checkbox. --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#github/googleapis/java-bigquery). --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5cd38742c..35bf5ecb9 100644 --- a/pom.xml +++ b/pom.xml @@ -54,7 +54,7 @@ github google-cloud-bigquery-parent v2-rev20220326-1.32.1 - 2.9.0 + 2.10.0     From 78903dbc6e6839c50c8f527190d0266882e75582 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Sat, 16 Apr 2022 06:20:10 +0200 Subject: [PATCH 191/277] chore(deps): update dependency com.google.cloud:google-cloud-bigqueryconnection to v2.2.3 (#1986) --- samples/install-without-bom/pom.xml | 2 +- samples/snapshot/pom.xml | 2 +- samples/snippets/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index 5e342ab79..0b58b476e 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -69,7 +69,7 @@ com.google.cloud google-cloud-bigqueryconnection - 2.2.2 + 2.2.3 test diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index eb3aa78bd..e86e6d5ec 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -67,7 +67,7 @@ com.google.cloud google-cloud-bigqueryconnection - 2.2.2 + 2.2.3 test diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index d75d36445..b95964164 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -85,7 +85,7 @@ com.google.cloud google-cloud-bigqueryconnection - 2.2.2 + 2.2.3 test From f989b3c4c43c50e6bb8c40ebc778cb7e9e091cad Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Sat, 16 Apr 2022 06:21:35 +0200 Subject: [PATCH 192/277] test(deps): update dependency com.google.cloud:google-cloud-datacatalog-bom to v1.7.3 (#1987) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 35bf5ecb9..d8692d33e 100644 --- a/pom.xml +++ b/pom.xml @@ -85,7 +85,7 @@ com.google.cloud google-cloud-datacatalog-bom - 1.7.2 + 1.7.3 pom import From 7dfcda1a1503a412620e0a985806675ad4dd0405 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Sat, 16 Apr 2022 04:36:21 +0000 Subject: [PATCH 193/277] chore(main): release 2.10.9 (#1984) :robot: I have created a release *beep* *boop* --- ### [2.10.9](https://github.com/googleapis/java-bigquery/compare/v2.10.8...v2.10.9) (2022-04-16) ### Dependencies * update dependency com.google.cloud:google-cloud-shared-dependencies to v2.10.0 ([#1983](https://github.com/googleapis/java-bigquery/issues/1983)) ([50ac31c](https://github.com/googleapis/java-bigquery/commit/50ac31c598ae8c5aa4e1f6dcde80da704db904a5)) * update dependency com.google.cloud:native-image-support to v0.13.1 ([#1982](https://github.com/googleapis/java-bigquery/issues/1982)) ([04f4679](https://github.com/googleapis/java-bigquery/commit/04f4679fe376e726b21e313115bfd48adc6cbe6b)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- CHANGELOG.md | 8 ++++++++ benchmark/pom.xml | 2 +- google-cloud-bigquery/pom.xml | 4 ++-- pom.xml | 4 ++-- samples/snapshot/pom.xml | 2 +- versions.txt | 2 +- 6 files changed, 15 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 499427191..d49ced0dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +### [2.10.9](https://github.com/googleapis/java-bigquery/compare/v2.10.8...v2.10.9) (2022-04-16) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-shared-dependencies to v2.10.0 ([#1983](https://github.com/googleapis/java-bigquery/issues/1983)) ([50ac31c](https://github.com/googleapis/java-bigquery/commit/50ac31c598ae8c5aa4e1f6dcde80da704db904a5)) +* update dependency com.google.cloud:native-image-support to v0.13.1 ([#1982](https://github.com/googleapis/java-bigquery/issues/1982)) ([04f4679](https://github.com/googleapis/java-bigquery/commit/04f4679fe376e726b21e313115bfd48adc6cbe6b)) + ### [2.10.8](https://github.com/googleapis/java-bigquery/compare/v2.10.7...v2.10.8) (2022-04-14) diff --git a/benchmark/pom.xml b/benchmark/pom.xml index e9466fcb9..2829ff763 100644 --- a/benchmark/pom.xml +++ b/benchmark/pom.xml @@ -6,7 +6,7 @@ google-cloud-bigquery-parent com.google.cloud - 2.10.9-SNAPSHOT + 2.10.9 diff --git a/google-cloud-bigquery/pom.xml b/google-cloud-bigquery/pom.xml index 81fc9437f..c7c17d455 100644 --- a/google-cloud-bigquery/pom.xml +++ b/google-cloud-bigquery/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-bigquery - 2.10.9-SNAPSHOT + 2.10.9 jar BigQuery https://github.com/googleapis/java-bigquery @@ -11,7 +11,7 @@ com.google.cloud google-cloud-bigquery-parent - 2.10.9-SNAPSHOT + 2.10.9 google-cloud-bigquery diff --git a/pom.xml b/pom.xml index d8692d33e..64174102f 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-bigquery-parent pom - 2.10.9-SNAPSHOT + 2.10.9 BigQuery Parent https://github.com/googleapis/java-bigquery @@ -99,7 +99,7 @@ com.google.cloud google-cloud-bigquery - 2.10.9-SNAPSHOT + 2.10.9 diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index e86e6d5ec..09c41694c 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -44,7 +44,7 @@ com.google.cloud google-cloud-bigquery - 2.10.9-SNAPSHOT + 2.10.9 diff --git a/versions.txt b/versions.txt index b771ef888..aa4bc48eb 100644 --- a/versions.txt +++ b/versions.txt @@ -1,4 +1,4 @@ # Format: # module:released-version:current-version -google-cloud-bigquery:2.10.8:2.10.9-SNAPSHOT \ No newline at end of file +google-cloud-bigquery:2.10.9:2.10.9 \ No newline at end of file From abe9b7e79dcfbe9eeb1bc18851b00d688582f721 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Sat, 16 Apr 2022 04:50:20 +0000 Subject: [PATCH 194/277] chore(main): release 2.10.10-SNAPSHOT (#1988) :robot: I have created a release *beep* *boop* --- ### Updating meta-information for bleeding-edge SNAPSHOT release. --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- benchmark/pom.xml | 2 +- google-cloud-bigquery/pom.xml | 4 ++-- pom.xml | 4 ++-- samples/snapshot/pom.xml | 2 +- versions.txt | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/benchmark/pom.xml b/benchmark/pom.xml index 2829ff763..ba3e3e0a1 100644 --- a/benchmark/pom.xml +++ b/benchmark/pom.xml @@ -6,7 +6,7 @@ google-cloud-bigquery-parent com.google.cloud - 2.10.9 + 2.10.10-SNAPSHOT diff --git a/google-cloud-bigquery/pom.xml b/google-cloud-bigquery/pom.xml index c7c17d455..666db0512 100644 --- a/google-cloud-bigquery/pom.xml +++ b/google-cloud-bigquery/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-bigquery - 2.10.9 + 2.10.10-SNAPSHOT jar BigQuery https://github.com/googleapis/java-bigquery @@ -11,7 +11,7 @@ com.google.cloud google-cloud-bigquery-parent - 2.10.9 + 2.10.10-SNAPSHOT google-cloud-bigquery diff --git a/pom.xml b/pom.xml index 64174102f..242712603 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-bigquery-parent pom - 2.10.9 + 2.10.10-SNAPSHOT BigQuery Parent https://github.com/googleapis/java-bigquery @@ -99,7 +99,7 @@ com.google.cloud google-cloud-bigquery - 2.10.9 + 2.10.10-SNAPSHOT diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 09c41694c..82f7cf3d8 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -44,7 +44,7 @@ com.google.cloud google-cloud-bigquery - 2.10.9 + 2.10.10-SNAPSHOT diff --git a/versions.txt b/versions.txt index aa4bc48eb..4f7ca68b9 100644 --- a/versions.txt +++ b/versions.txt @@ -1,4 +1,4 @@ # Format: # module:released-version:current-version -google-cloud-bigquery:2.10.9:2.10.9 \ No newline at end of file +google-cloud-bigquery:2.10.9:2.10.10-SNAPSHOT \ No newline at end of file From 00c9e4016ef9164345e76dacac4e91fc8e79a4b8 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Mon, 18 Apr 2022 17:40:39 +0200 Subject: [PATCH 195/277] chore(deps): update dependency com.google.cloud:google-cloud-bigquery to v2.10.9 (#1989) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [![WhiteSource Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com) This PR contains the following updates: | Package | Change | Age | Adoption | Passing | Confidence | |---|---|---|---|---|---| | [com.google.cloud:google-cloud-bigquery](https://togithub.com/googleapis/java-bigquery) | `2.10.8` -> `2.10.9` | [![age](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-bigquery/2.10.9/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-bigquery/2.10.9/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-bigquery/2.10.9/compatibility-slim/2.10.8)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-bigquery/2.10.9/confidence-slim/2.10.8)](https://docs.renovatebot.com/merge-confidence/) | --- ### Release Notes
    googleapis/java-bigquery ### [`v2.10.9`](https://togithub.com/googleapis/java-bigquery/blob/HEAD/CHANGELOG.md#​2109-httpsgithubcomgoogleapisjava-bigquerycomparev2108v2109-2022-04-16) [Compare Source](https://togithub.com/googleapis/java-bigquery/compare/v2.10.8...v2.10.9)
    --- ### Configuration 📅 **Schedule**: At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Renovate will not automatically rebase this PR, because other commits have been found. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, click this checkbox. --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#github/googleapis/java-bigquery). --- README.md | 6 +++--- samples/install-without-bom/pom.xml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index f0e0c1053..0616bfdbd 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ If you are using Maven without BOM, add this to your dependencies: com.google.cloud google-cloud-bigquery - 2.10.8 + 2.10.9 ``` @@ -59,13 +59,13 @@ implementation 'com.google.cloud:google-cloud-bigquery' If you are using Gradle without BOM, add this to your dependencies ```Groovy -implementation 'com.google.cloud:google-cloud-bigquery:2.10.8' +implementation 'com.google.cloud:google-cloud-bigquery:2.10.9' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.10.8" +libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.10.9" ``` ## Authentication diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index 0b58b476e..85acf0b23 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -45,7 +45,7 @@ com.google.cloud google-cloud-bigquery - 2.10.8 + 2.10.9 From 917b3eacf2b9ca9066347508cb9a5fc7fd24075b Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Mon, 18 Apr 2022 23:14:26 +0200 Subject: [PATCH 196/277] deps: update dependency com.google.cloud:google-cloud-storage to v2.6.1 (#1991) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [![WhiteSource Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com) This PR contains the following updates: | Package | Change | Age | Adoption | Passing | Confidence | |---|---|---|---|---|---| | [com.google.cloud:google-cloud-storage](https://togithub.com/googleapis/java-storage) | `2.6.0` -> `2.6.1` | [![age](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-storage/2.6.1/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-storage/2.6.1/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-storage/2.6.1/compatibility-slim/2.6.0)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-storage/2.6.1/confidence-slim/2.6.0)](https://docs.renovatebot.com/merge-confidence/) | --- ### Release Notes
    googleapis/java-storage ### [`v2.6.1`](https://togithub.com/googleapis/java-storage/blob/HEAD/CHANGELOG.md#​261-httpsgithubcomgoogleapisjava-storagecomparev260v261-2022-04-15) [Compare Source](https://togithub.com/googleapis/java-storage/compare/v2.6.0...v2.6.1)
    --- ### Configuration 📅 **Schedule**: At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, click this checkbox. --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#github/googleapis/java-bigquery). --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 242712603..66921d200 100644 --- a/pom.xml +++ b/pom.xml @@ -157,7 +157,7 @@ com.google.cloud google-cloud-storage - 2.6.0 + 2.6.1 test From d864514568c1f59e966a327bbd5a01c6c5d1bbff Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Mon, 18 Apr 2022 23:14:30 +0200 Subject: [PATCH 197/277] deps: update dependency com.google.cloud:google-cloud-bigtable to v2.6.2 (#1990) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [![WhiteSource Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com) This PR contains the following updates: | Package | Change | Age | Adoption | Passing | Confidence | |---|---|---|---|---|---| | [com.google.cloud:google-cloud-bigtable](https://togithub.com/googleapis/java-bigtable) | `2.6.1` -> `2.6.2` | [![age](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-bigtable/2.6.2/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-bigtable/2.6.2/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-bigtable/2.6.2/compatibility-slim/2.6.1)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-bigtable/2.6.2/confidence-slim/2.6.1)](https://docs.renovatebot.com/merge-confidence/) | --- ### Release Notes
    googleapis/java-bigtable ### [`v2.6.2`](https://togithub.com/googleapis/java-bigtable/blob/HEAD/CHANGELOG.md#​262-httpsgithubcomgoogleapisjava-bigtablecomparev261v262-2022-04-15) [Compare Source](https://togithub.com/googleapis/java-bigtable/compare/v2.6.1...v2.6.2)
    --- ### Configuration 📅 **Schedule**: At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, click this checkbox. --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#github/googleapis/java-bigquery). --- samples/install-without-bom/pom.xml | 2 +- samples/snapshot/pom.xml | 2 +- samples/snippets/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index 85acf0b23..24a0de472 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -63,7 +63,7 @@ com.google.cloud google-cloud-bigtable - 2.6.1 + 2.6.2 test diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 82f7cf3d8..1057e8793 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -61,7 +61,7 @@ com.google.cloud google-cloud-bigtable - 2.6.1 + 2.6.2 test diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index b95964164..25edf810d 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -79,7 +79,7 @@ com.google.cloud google-cloud-bigtable - 2.6.1 + 2.6.2 test From 1d262dc58d1b5dd06d138fcd72ba060f170b95bd Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 18 Apr 2022 21:36:15 +0000 Subject: [PATCH 198/277] chore(main): release 2.10.10 (#1992) :robot: I have created a release *beep* *boop* --- * update dependency com.google.cloud:google-cloud-bigtable to v2.6.2 ([#1990](https://github.com/googleapis/java-bigquery/issues/1990)) ([4c1cb4c](https://github.com/googleapis/java-bigquery/commit/4c1cb4c13214556b706f1ff8c50a46f881bf2724)) * update dependency com.google.cloud:google-cloud-storage to v2.6.1 ([#1991](https://github.com/googleapis/java-bigquery/issues/1991)) ([e02bf31](https://github.com/googleapis/java-bigquery/commit/e02bf315737dba50741c1346af8bde6871cb857a)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). chore(main): release 2.10.11-SNAPSHOT (#1993) :robot: I have created a release *beep* *boop* --- --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- CHANGELOG.md | 8 ++++++++ benchmark/pom.xml | 2 +- google-cloud-bigquery/pom.xml | 4 ++-- pom.xml | 4 ++-- samples/snapshot/pom.xml | 2 +- versions.txt | 2 +- 6 files changed, 15 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d49ced0dc..f17245708 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +### [2.10.10](https://github.com/googleapis/java-bigquery/compare/v2.10.9...v2.10.10) (2022-04-18) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-bigtable to v2.6.2 ([#1990](https://github.com/googleapis/java-bigquery/issues/1990)) ([4c1cb4c](https://github.com/googleapis/java-bigquery/commit/4c1cb4c13214556b706f1ff8c50a46f881bf2724)) +* update dependency com.google.cloud:google-cloud-storage to v2.6.1 ([#1991](https://github.com/googleapis/java-bigquery/issues/1991)) ([e02bf31](https://github.com/googleapis/java-bigquery/commit/e02bf315737dba50741c1346af8bde6871cb857a)) + ### [2.10.9](https://github.com/googleapis/java-bigquery/compare/v2.10.8...v2.10.9) (2022-04-16) diff --git a/benchmark/pom.xml b/benchmark/pom.xml index ba3e3e0a1..10f47165e 100644 --- a/benchmark/pom.xml +++ b/benchmark/pom.xml @@ -6,7 +6,7 @@ google-cloud-bigquery-parent com.google.cloud - 2.10.10-SNAPSHOT + 2.10.11-SNAPSHOT diff --git a/google-cloud-bigquery/pom.xml b/google-cloud-bigquery/pom.xml index 666db0512..0e371fda6 100644 --- a/google-cloud-bigquery/pom.xml +++ b/google-cloud-bigquery/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-bigquery - 2.10.10-SNAPSHOT + 2.10.11-SNAPSHOT jar BigQuery https://github.com/googleapis/java-bigquery @@ -11,7 +11,7 @@ com.google.cloud google-cloud-bigquery-parent - 2.10.10-SNAPSHOT + 2.10.11-SNAPSHOT google-cloud-bigquery diff --git a/pom.xml b/pom.xml index 66921d200..f3c958f18 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-bigquery-parent pom - 2.10.10-SNAPSHOT + 2.10.11-SNAPSHOT BigQuery Parent https://github.com/googleapis/java-bigquery @@ -99,7 +99,7 @@ com.google.cloud google-cloud-bigquery - 2.10.10-SNAPSHOT + 2.10.11-SNAPSHOT diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 1057e8793..2d7044933 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -44,7 +44,7 @@ com.google.cloud google-cloud-bigquery - 2.10.10-SNAPSHOT + 2.10.11-SNAPSHOT diff --git a/versions.txt b/versions.txt index 4f7ca68b9..ad1668e67 100644 --- a/versions.txt +++ b/versions.txt @@ -1,4 +1,4 @@ # Format: # module:released-version:current-version -google-cloud-bigquery:2.10.9:2.10.10-SNAPSHOT \ No newline at end of file +google-cloud-bigquery:2.10.10:2.10.11-SNAPSHOT \ No newline at end of file From ac03e540672f29719a8588b8cd4ed77a8c6ca8fa Mon Sep 17 00:00:00 2001 From: stephwang Date: Mon, 18 Apr 2022 19:56:09 -0400 Subject: [PATCH 199/277] fix dependencies.sh --- .kokoro/dependencies.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.kokoro/dependencies.sh b/.kokoro/dependencies.sh index 586b78bb9..545820622 100755 --- a/.kokoro/dependencies.sh +++ b/.kokoro/dependencies.sh @@ -67,13 +67,13 @@ function completenessCheck() { msg "Generating dependency list using original pom..." # Excluding commons-codec,commons-logging from the comparison as a temp fix # Explanation and issue filed in maven-dependency-plugin: https://issues.apache.org/jira/browse/MDEP-737 - mvn dependency:list -f pom.xml -DexcludeArtifactIds=commons-codec,commons-logging -DincludeScope=runtime -Dsort=true | grep '\[INFO] .*:.*:.*:.*:.*' | sed -e s/\\s--\\smodule.*// >.org-list.txt + mvn dependency:list -f pom.xml -DexcludeArtifactIds=commons-codec,commons-logging,grpc-googleapis -DincludeScope=runtime -Dsort=true | grep '\[INFO] .*:.*:.*:.*:.*' | sed -e s/\\s--\\smodule.*// >.org-list.txt # Output dep list generated using the flattened pom (only 'compile' and 'runtime' scopes) msg "Generating dependency list using flattened pom..." # Excluding commons-codec,commons-logging from the comparison as a temp fix # Explanation and issue filed in maven-dependency-plugin: https://issues.apache.org/jira/browse/MDEP-737 - mvn dependency:list -f .flattened-pom.xml -DexcludeArtifactIds=commons-codec,commons-logging -DincludeScope=runtime -Dsort=true | grep '\[INFO] .*:.*:.*:.*:.*' >.new-list.txt + mvn dependency:list -f .flattened-pom.xml -DexcludeArtifactIds=commons-codec,commons-logging,grpc-googleapis -DincludeScope=runtime -Dsort=true | grep '\[INFO] .*:.*:.*:.*:.*' >.new-list.txt # Compare two dependency lists msg "Comparing dependency lists..." From e33f705c5016a37ca6494e540656b5b3b1e2534a Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Tue, 19 Apr 2022 00:13:29 +0000 Subject: [PATCH 200/277] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot?= =?UTF-8?q?=20post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0616bfdbd..be04041b2 100644 --- a/README.md +++ b/README.md @@ -59,13 +59,13 @@ implementation 'com.google.cloud:google-cloud-bigquery' If you are using Gradle without BOM, add this to your dependencies ```Groovy -implementation 'com.google.cloud:google-cloud-bigquery:2.10.9' +implementation 'com.google.cloud:google-cloud-bigquery:2.10.10' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.10.9" +libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.10.10" ``` ## Authentication From c9b02adb9849f8bfba1e7269a25cf6cc57a84b44 Mon Sep 17 00:00:00 2001 From: stephwang Date: Mon, 18 Apr 2022 22:13:56 -0400 Subject: [PATCH 201/277] update query parameters for dryRun --- .../cloud/bigquery/BigQueryDryRunResult.java | 3 +-- .../cloud/bigquery/BigQueryDryRunResultImpl.java | 7 +++---- .../com/google/cloud/bigquery/ConnectionImpl.java | 15 +++++++++++++-- .../google/cloud/bigquery/it/ITBigQueryTest.java | 10 +++------- 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResult.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResult.java index 99f281b16..0cc6b0891 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResult.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResult.java @@ -16,7 +16,6 @@ package com.google.cloud.bigquery; -import com.google.api.services.bigquery.model.QueryParameter; import java.util.List; public interface BigQueryDryRunResult { @@ -25,5 +24,5 @@ public interface BigQueryDryRunResult { Schema getSchema() throws BigQuerySQLException; /** Returns query parameters for standard SQL queries */ - List getQueryParameters() throws BigQuerySQLException; + List getQueryParameters() throws BigQuerySQLException; } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResultImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResultImpl.java index 76d9174ec..e8724f223 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResultImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResultImpl.java @@ -16,15 +16,14 @@ package com.google.cloud.bigquery; -import com.google.api.services.bigquery.model.QueryParameter; import java.util.List; public class BigQueryDryRunResultImpl implements BigQueryDryRunResult { private Schema schema; - private List queryParameters; + private List queryParameters; BigQueryDryRunResultImpl( - Schema schema, List queryParameters) { // Package-Private access + Schema schema, List queryParameters) { // Package-Private access this.schema = schema; this.queryParameters = queryParameters; } @@ -35,7 +34,7 @@ public Schema getSchema() throws BigQuerySQLException { } @Override - public List getQueryParameters() throws BigQuerySQLException { + public List getQueryParameters() throws BigQuerySQLException { return queryParameters; } } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 789fe3ea4..aebc473c2 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -121,7 +121,7 @@ public synchronized Boolean cancel() throws BigQuerySQLException { * This method runs a dry run query * * @param sql SQL SELECT statement - * @return BigQueryDryRunResult containing List and Schema + * @return BigQueryDryRunResult containing List and Schema * @throws BigQuerySQLException */ @BetaApi @@ -129,8 +129,10 @@ public synchronized Boolean cancel() throws BigQuerySQLException { public BigQueryDryRunResult dryRun(String sql) throws BigQuerySQLException { com.google.api.services.bigquery.model.Job dryRunJob = createDryRunJob(sql); Schema schema = Schema.fromPb(dryRunJob.getStatistics().getQuery().getSchema()); - List queryParameters = + List queryParametersPb = dryRunJob.getStatistics().getQuery().getUndeclaredQueryParameters(); + List queryParameters = + Lists.transform(queryParametersPb, QUERY_PARAMETER_FROM_PB_FUNCTION); return new BigQueryDryRunResultImpl(schema, queryParameters); } @@ -1197,4 +1199,13 @@ private com.google.api.services.bigquery.model.Job createDryRunJob(String sql) { queryParameterPb.setParameterType(value.getQueryParameterValue().toTypePb()); return queryParameterPb; }; + + // Convert from QueryParameter class to the Parameter wrapper class + private static final Function QUERY_PARAMETER_FROM_PB_FUNCTION = + pb -> + Parameter.newBuilder() + .setName(pb.getName() == null ? "" : pb.getName()) + .setQueryParameterValue( + QueryParameterValue.fromPb(pb.getParameterValue(), pb.getParameterType())) + .build(); } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 23e27bac3..c3757e811 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -29,8 +29,6 @@ import static org.junit.Assert.fail; import com.google.api.gax.paging.Page; -import com.google.api.services.bigquery.model.QueryParameter; -import com.google.api.services.bigquery.model.QueryParameterType; import com.google.auth.oauth2.GoogleCredentials; import com.google.auth.oauth2.ServiceAccountCredentials; import com.google.cloud.Date; @@ -2495,11 +2493,9 @@ public void testConnectionImplDryRun() throws SQLException { assertNotNull(bigQueryDryRunResultSet.getSchema()); assertEquals( BQ_RESULTSET_EXPECTED_SCHEMA, bigQueryDryRunResultSet.getSchema()); // match the schema - List queryParameters = bigQueryDryRunResultSet.getQueryParameters(); - List expectedQueryParameters = - ImmutableList.of( - new QueryParameter().setParameterType(new QueryParameterType().setType("STRING"))); - assertEquals(expectedQueryParameters, queryParameters); + List queryParameters = bigQueryDryRunResultSet.getQueryParameters(); + assertEquals( + StandardSQLTypeName.STRING, queryParameters.get(0).getQueryParameterValue().getType()); } @Test From e5be7243e54ce0f35460cda8903f798c2b064446 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 20 Apr 2022 14:58:04 +0530 Subject: [PATCH 202/277] Made Label optional by making it a vararg --- .../com/google/cloud/bigquery/Connection.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java index bf0740ae9..f14f89db1 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java @@ -75,22 +75,22 @@ public interface Connection { BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException; /** - * Execute a SQL statement with query parameters that returns a single ResultSet + * This method executes a SQL SELECT query * - * @param sql typically a static SQL SELECT statement + * @param sql SQL SELECT query * @param parameters named or positional parameters. The set of query parameters must either be - * all positional or all named parameters. Named parameters are denoted using an @ prefix, - * e.g. @myParam for a parameter named "myParam". + * all positional or all named parameters. * @param labels the labels associated with this query. You can use these to organize and group * your query jobs. Label keys and values can be no longer than 63 characters, can only * contain lowercase letters, numeric characters, underscores and dashes. International - * characters are allowed. Label values are optional. Label keys must start with a letter and - * each label in the list must have a different key. - * @return a ResultSet that contains the data produced by the query - * @exception BigQuerySQLException if a database access error occurs + * characters are allowed. Label values are optional and Label is a Varargs. You should pass + * all the Labels in a single Map .Label keys must start with a letter and each label in the + * list must have a different key. + * @return BigQueryResultSet containing the output of the query + * @throws BigQuerySQLException */ @BetaApi BigQueryResultSet executeSelect( - String sql, List parameters, Map labels) + String sql, List parameters, Map... labels) throws BigQuerySQLException; } From c6f6883455ba47707435701a9da7f92d3e837949 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 20 Apr 2022 15:01:00 +0530 Subject: [PATCH 203/277] Made Label optional by making it a vararg --- .../google/cloud/bigquery/ConnectionImpl.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index aebc473c2..ea12130cf 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -173,27 +173,33 @@ public BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException { * @param labels the labels associated with this query. You can use these to organize and group * your query jobs. Label keys and values can be no longer than 63 characters, can only * contain lowercase letters, numeric characters, underscores and dashes. International - * characters are allowed. Label values are optional. Label keys must start with a letter and - * each label in the list must have a different key. + * characters are allowed. Label values are optional and Label is a Varargs. You should pass + * all the Labels in a single Map .Label keys must start with a letter and each label in the + * list must have a different key. * @return BigQueryResultSet containing the output of the query * @throws BigQuerySQLException */ @BetaApi @Override public BigQueryResultSet executeSelect( - String sql, List parameters, Map labels) + String sql, List parameters, Map... labels) throws BigQuerySQLException { + Map labelMap = null; + if (labels != null + && labels.length == 1) { // We expect label as a key value pair in a single Map + labelMap = labels[0]; + } try { // use jobs.query if possible if (isFastQuerySupported()) { final String projectId = bigQueryOptions.getProjectId(); final QueryRequest queryRequest = - createQueryRequest(connectionSettings, sql, parameters, labels); + createQueryRequest(connectionSettings, sql, parameters, labelMap); return queryRpc(projectId, queryRequest, parameters != null); } // use jobs.insert otherwise com.google.api.services.bigquery.model.Job queryJob = - createQueryJob(sql, connectionSettings, parameters, labels); + createQueryJob(sql, connectionSettings, parameters, labelMap); JobId jobId = JobId.fromPb(queryJob.getJobReference()); GetQueryResultsResponse firstPage = getQueryResultsFirstPage(jobId); return getResultSet(firstPage, jobId, sql, parameters != null); From ffa3be0390bde01849c5b379ede7b56c92a0b738 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 20 Apr 2022 15:02:25 +0530 Subject: [PATCH 204/277] Removed null labels --- .../java/com/google/cloud/bigquery/it/ITBigQueryTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index c3757e811..d255c041a 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -3371,7 +3371,7 @@ public void testExecuteSelectWithPositionalQueryParameters() throws BigQuerySQLE ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); Connection connection = bigquery.createConnection(connectionSettings); List parameters = ImmutableList.of(stringParam, timeStampParam); - BigQueryResultSet rs = connection.executeSelect(query, parameters, null); + BigQueryResultSet rs = connection.executeSelect(query, parameters); assertEquals(2, rs.getTotalRows()); } @@ -3422,7 +3422,7 @@ public void testExecuteSelectWithNamedQueryParameters() throws BigQuerySQLExcept ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); Connection connection = bigquery.createConnection(connectionSettings); List parameters = ImmutableList.of(stringParam, intArrayParam); - BigQueryResultSet rs = connection.executeSelect(query, parameters, null); + BigQueryResultSet rs = connection.executeSelect(query, parameters); assertEquals(2, rs.getTotalRows()); } From bcdaa6df6e7ef54d1b585d4289f92cfa309ce1ee Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 21 Apr 2022 11:05:37 +0530 Subject: [PATCH 205/277] Renamed cancel to close --- .../src/main/java/com/google/cloud/bigquery/Connection.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java index f14f89db1..036b52ddc 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java @@ -28,7 +28,7 @@ public interface Connection { /** Sends a query cancel request. This call will return immediately */ @BetaApi - Boolean cancel() throws BigQuerySQLException; + Boolean close() throws BigQuerySQLException; /** * Execute a query dry run that does not return any BigQueryResultSet // TODO: explain more about From eb3733284ff5ae3a57f64dfd1ffbc86e9cd189b9 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 21 Apr 2022 11:15:24 +0530 Subject: [PATCH 206/277] Refactored testConnectionCancel --- .../java/com/google/cloud/bigquery/it/ITBigQueryTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index d255c041a..4ce506ef5 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -2617,7 +2617,7 @@ public void testExecuteSelectSinglePageTableRow() throws SQLException { } @Test - public void testConnectionCancel() throws SQLException { + public void testConnectionClose() throws SQLException { String query = "SELECT date, county, state_name, confirmed_cases, deaths FROM " + TABLE_ID_LARGE.getTable() @@ -2634,7 +2634,7 @@ public void testConnectionCancel() throws SQLException { while (rs.next()) { ++cnt; if (cnt > 57000) { // breaking at 57K, query reads 300K - assertTrue(connection.cancel()); // we should be able to cancel the connection + assertTrue(connection.close()); // we should be able to cancel the connection } } assertTrue( From 1b03be8e4f39423cdc8a8a487184af597aef0072 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 21 Apr 2022 11:16:22 +0530 Subject: [PATCH 207/277] Renamed cancel() to close() --- .../src/main/java/com/google/cloud/bigquery/ConnectionImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index ea12130cf..f57c4b4e0 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -112,7 +112,7 @@ class ConnectionImpl implements Connection { */ @BetaApi @Override - public synchronized Boolean cancel() throws BigQuerySQLException { + public synchronized Boolean close() throws BigQuerySQLException { queryTaskExecutor.shutdownNow(); return queryTaskExecutor.isShutdown(); } From 48d99d5d71eaee31bb26c366c7c9cb6e7147dead Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 21 Apr 2022 11:17:14 +0530 Subject: [PATCH 208/277] Renamed testCancel --- .../java/com/google/cloud/bigquery/ConnectionImplTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java index 06620d8d2..706264683 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java @@ -172,8 +172,8 @@ public void testFastQueryMultiplePages() throws BigQuerySQLException { } @Test - public void testCancel() throws BigQuerySQLException { - boolean cancelled = connection.cancel(); + public void testClose() throws BigQuerySQLException { + boolean cancelled = connection.close(); assertTrue(cancelled); } From 8cbef7855f59fde46910b47bcc1fccf94b238ed3 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 21 Apr 2022 15:47:13 +0530 Subject: [PATCH 209/277] Added testPositionalParams --- .../bigquery/it/ITNightlyBigQueryTest.java | 53 +++++++++++++++++-- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java index e5ff18826..b929ad036 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java @@ -36,7 +36,9 @@ import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.InsertAllRequest; import com.google.cloud.bigquery.InsertAllResponse; +import com.google.cloud.bigquery.Parameter; import com.google.cloud.bigquery.QueryJobConfiguration; +import com.google.cloud.bigquery.QueryParameterValue; import com.google.cloud.bigquery.ReadClientConnectionConfiguration; import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.StandardSQLTypeName; @@ -46,6 +48,7 @@ import com.google.cloud.bigquery.TableId; import com.google.cloud.bigquery.TableInfo; import com.google.cloud.bigquery.testing.RemoteBigQueryHelper; +import com.google.common.collect.ImmutableList; import com.google.common.io.BaseEncoding; import java.io.IOException; import java.math.BigDecimal; @@ -89,7 +92,7 @@ public class ITNightlyBigQueryTest { + " JSONField, JSONField.hello, JSONField.id from %s.%s order by IntegerField asc LIMIT %s"; private static final String POSITIONAL_QUERY = String.format( - "select StringField, GeographyField, BooleanField, DateField from %s.%s where DateField = ? LIMIT %s", + "select RecordField.BooleanField, RecordField.StringField, StringField, BooleanField, BytesField, IntegerField, GeographyField, NumericField, BigNumericField, TimeField, DateField, TimestampField, JSONField from %s.%s where DateField = ? and BooleanField = ? and IntegerField > ? and NumericField > ? LIMIT %s", DATASET, TABLE, MULTI_LIMIT_RECS); private static final String QUERY = String.format(BASE_QUERY, DATASET, TABLE, LIMIT_RECS); private static final String MULTI_QUERY = @@ -202,7 +205,7 @@ public void testInvalidQuery() throws BigQuerySQLException { assertNotNull(ex.getMessage()); assertTrue(ex.getMessage().toLowerCase().contains("unexpected keyword into")); } finally { - connection.cancel(); + connection.close(); } } @@ -264,7 +267,7 @@ public void testIterateAndOrder() throws SQLException { ++cnt; } assertEquals(LIMIT_RECS, cnt); // all the records were retrieved - connection.cancel(); + connection.close(); } @Test @@ -311,7 +314,7 @@ public void testMultipleRuns() throws SQLException { } ++cnt; } - connection.cancel(); + connection.close(); totalCnt += cnt; // Repeat the same run connection = getConnection(); @@ -352,11 +355,51 @@ public void testMultipleRuns() throws SQLException { } ++cnt; } - connection.cancel(); + connection.close(); totalCnt += cnt; assertEquals(MULTI_LIMIT_RECS * 2, totalCnt); } + @Test + public void testPositionalParams() + throws SQLException { // Bypasses Read API as it doesnt support Positional Params + Connection connection = getConnection(); + Parameter dateParam = + Parameter.newBuilder() + .setQueryParameterValue(QueryParameterValue.date("2022-01-01")) + .build(); + Parameter boolParam = + Parameter.newBuilder().setQueryParameterValue(QueryParameterValue.bool(true)).build(); + Parameter intParam = + Parameter.newBuilder().setQueryParameterValue(QueryParameterValue.int64(1)).build(); + Parameter numericParam = + Parameter.newBuilder() + .setQueryParameterValue(QueryParameterValue.numeric(new BigDecimal(100))) + .build(); + List parameters = ImmutableList.of(dateParam, boolParam, intParam, numericParam); + + BigQueryResultSet bigQueryResultSet = connection.executeSelect(POSITIONAL_QUERY, parameters); + logger.log(Level.INFO, "Query used: {0}", POSITIONAL_QUERY); + ResultSet rs = bigQueryResultSet.getResultSet(); + int cnt = 0; + while (rs.next()) { + assertFalse(rs.getBoolean("BooleanField")); + assertTrue(0.0d <= rs.getDouble("BigNumericField")); + assertTrue(0 <= rs.getInt("IntegerField")); + assertTrue(0L <= rs.getLong("NumericField")); + assertNotNull(rs.getBytes("BytesField")); + assertNotNull(rs.getTimestamp("TimestampField")); + assertNotNull(rs.getTime("TimeField")); + assertNotNull(rs.getDate("DateField")); + assertNotNull(rs.getString("JSONField")); + assertTrue(rs.getBoolean("BooleanField_1")); + assertNotNull(rs.getString("StringField_1")); + ++cnt; + } + connection.close(); + assertEquals(MULTI_LIMIT_RECS, cnt); + } + // asserts the value of each row private static void testForAllDataTypeValues(ResultSet rs, int cnt) throws SQLException { // Testing JSON type From 251eace3665769287bbba9c8112de083783c1c43 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 26 Apr 2022 14:59:05 +0530 Subject: [PATCH 210/277] Replaced all the shutdownNows with MoreExecutors.shutdownAndAwaitTermination() --- .../google/cloud/bigquery/ConnectionImpl.java | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index f57c4b4e0..a27edcf56 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -44,6 +44,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.util.concurrent.MoreExecutors; import java.io.IOException; import java.util.AbstractList; import java.util.ArrayList; @@ -57,6 +58,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -83,6 +85,7 @@ class ConnectionImpl implements Connection { Executors.newFixedThreadPool(MAX_PROCESS_QUERY_THREADS_CNT); private final Logger logger = Logger.getLogger(this.getClass().getName()); private BigQueryReadClient bqReadClient; + private static final long EXECUTOR_TIMEOUT_SEC = 30; ConnectionImpl( ConnectionSettings connectionSettings, @@ -113,8 +116,8 @@ class ConnectionImpl implements Connection { @BetaApi @Override public synchronized Boolean close() throws BigQuerySQLException { - queryTaskExecutor.shutdownNow(); - return queryTaskExecutor.isShutdown(); + return MoreExecutors.shutdownAndAwaitTermination( + queryTaskExecutor, EXECUTOR_TIMEOUT_SEC, TimeUnit.SECONDS); } /** @@ -520,7 +523,10 @@ void populateBufferAsync( } catch (InterruptedException e) { throw new BigQueryException(0, e.getMessage(), e); } finally { - queryTaskExecutor.shutdownNow(); // Shutdown the thread pool + MoreExecutors.shutdownAndAwaitTermination( + queryTaskExecutor, + EXECUTOR_TIMEOUT_SEC, + TimeUnit.SECONDS); // Shutdown the thread pool } }; @@ -763,7 +769,10 @@ private void processArrowStreamAsync( } finally { try { buffer.put(Tuple.of(null, false)); // marking end of stream - queryTaskExecutor.shutdownNow(); // Shutdown the thread pool + MoreExecutors.shutdownAndAwaitTermination( + queryTaskExecutor, + EXECUTOR_TIMEOUT_SEC, + TimeUnit.SECONDS); // Shutdown the thread pool } catch (InterruptedException e) { logger.log(Level.WARNING, "\n Error occurred ", e); } From 28d620cfa9f6638ee5b7eb421f6eb5173cef5694 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 26 Apr 2022 15:18:57 +0530 Subject: [PATCH 211/277] Using TimeUnit.DAYS.toMillis for conversion from Days to Millis --- .../com/google/cloud/bigquery/BigQueryResultSetImpl.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index 22307996d..2176f1c36 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -29,6 +29,7 @@ import java.util.Map; import java.util.TimeZone; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; import org.apache.arrow.vector.util.JsonStringArrayList; import org.apache.arrow.vector.util.Text; @@ -525,9 +526,9 @@ public Date getDate(String fieldName) throws SQLException { } else { Integer dateInt = (Integer) dateObj; long dateInMillis = - Long.valueOf(dateInt) - * (24 * 60 * 60 - * 1000); // For example int 18993 represents 2022-01-01, converting time to + TimeUnit.DAYS.toMillis( + Long.valueOf( + dateInt)); // For example int 18993 represents 2022-01-01, converting time to // milli seconds return new Date(dateInMillis); } From d8af94fde3d98061932f373920e28769a9917ec2 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 26 Apr 2022 17:44:57 +0530 Subject: [PATCH 212/277] deleted outdated comment --- .../java/com/google/cloud/bigquery/BigQueryResultSetImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index 2176f1c36..c634d843b 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -33,7 +33,6 @@ import org.apache.arrow.vector.util.JsonStringArrayList; import org.apache.arrow.vector.util.Text; -// TODO: This implementation deals with the JSON response. We can have respective implementations public class BigQueryResultSetImpl implements BigQueryResultSet { private final Schema schema; From 57fd5e4874365e04f4fb0c5730d2607eceb6b9be Mon Sep 17 00:00:00 2001 From: stephwang Date: Tue, 26 Apr 2022 11:02:43 -0400 Subject: [PATCH 213/277] nit update --- .../java/com/google/cloud/bigquery/BigQueryResultSetImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index 2176f1c36..c634d843b 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -33,7 +33,6 @@ import org.apache.arrow.vector.util.JsonStringArrayList; import org.apache.arrow.vector.util.Text; -// TODO: This implementation deals with the JSON response. We can have respective implementations public class BigQueryResultSetImpl implements BigQueryResultSet { private final Schema schema; From 78259c515c00dcfca6932aebd33ce09d236409bf Mon Sep 17 00:00:00 2001 From: stephwang Date: Tue, 26 Apr 2022 17:10:57 -0400 Subject: [PATCH 214/277] update based on comments --- .../src/main/java/com/google/cloud/bigquery/BigQuery.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java index e83cb6f18..37288f9c8 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java @@ -761,7 +761,8 @@ public int hashCode() { Job create(JobInfo jobInfo, JobOption... options); /** - * Creates a new BigQuery query connection. + * Creates a new BigQuery query connection used for executing queries (not the same as BigQuery + * connection properties). * *

    Example of creating a query connection. * From 25eb4e567f6a9785f118a1900aa42b448237ebcb Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 27 Apr 2022 10:23:26 +0530 Subject: [PATCH 215/277] updated to using a public dataset --- .../main/java/com.google.cloud.bigquery/ConnImplBenchmark.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java b/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java index 8eef2e770..45b6fe20f 100644 --- a/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java +++ b/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java @@ -50,7 +50,8 @@ public class ConnImplBenchmark { private ConnectionSettings connectionSettingsReadAPIEnabled, connectionSettingsReadAPIDisabled; private long numBuffRows = 100000L; private final String DATASET = "bigquery_test_dataset"; - private final String QUERY = "SELECT * FROM tlc_yellow_trips_2017_stephwang LIMIT %s"; + private final String QUERY = + "SELECT * FROM bigquery-public-data.new_york_taxi_trips.tlc_yellow_trips_2017 LIMIT %s"; @Setup public void setUp() throws IOException { From e7b4cb659c0ceced25717a37127f3780cde6f990 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 27 Apr 2022 10:26:21 +0530 Subject: [PATCH 216/277] updated cancel() to close() --- .../java/com.google.cloud.bigquery/ConnImplBenchmark.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java b/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java index 45b6fe20f..94b38055b 100644 --- a/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java +++ b/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java @@ -102,7 +102,7 @@ public void iterateRecordsUsingReadAPI(Blackhole blackhole) } catch (Exception e) { e.printStackTrace(); } finally { - connectionReadAPIEnabled.cancel(); // IMP to kill the bg workers + connectionReadAPIEnabled.close(); // IMP to kill the bg workers } blackhole.consume(hash); } @@ -122,7 +122,7 @@ public void iterateRecordsWithoutUsingReadAPI(Blackhole blackhole) } catch (Exception e) { e.printStackTrace(); } finally { - connectionReadAPIDisabled.cancel(); // IMP to kill the bg workers + connectionReadAPIDisabled.close(); // IMP to kill the bg workers } blackhole.consume(hash); } From 1a7e51210b115ba63ba20448ffe8f533fe167964 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 27 Apr 2022 10:30:58 +0530 Subject: [PATCH 217/277] using constants for clientConnectionConfiguration --- .../java/com.google.cloud.bigquery/ConnImplBenchmark.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java b/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java index 94b38055b..2ebb4e90a 100644 --- a/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java +++ b/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java @@ -52,6 +52,8 @@ public class ConnImplBenchmark { private final String DATASET = "bigquery_test_dataset"; private final String QUERY = "SELECT * FROM bigquery-public-data.new_york_taxi_trips.tlc_yellow_trips_2017 LIMIT %s"; + public static final long NUM_PAGE_ROW_CNT_RATIO = 10; + public static final long NUM_MIN_RESULT_SIZE = 200000; @Setup public void setUp() throws IOException { @@ -60,8 +62,8 @@ public void setUp() throws IOException { clientConnectionConfiguration = ReadClientConnectionConfiguration.newBuilder() - .setTotalToPageRowCountRatio(10L) - .setMinResultSize(200000L) + .setTotalToPageRowCountRatio(NUM_PAGE_ROW_CNT_RATIO) + .setMinResultSize(NUM_MIN_RESULT_SIZE) .setBufferSize(numBuffRows) .build(); From 6a1f69d579f9d8189dfd576e058fbc1a6bc92f02 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 27 Apr 2022 10:32:27 +0530 Subject: [PATCH 218/277] using constants for clientConnectionConfiguration --- .../java/com.google.cloud.bigquery/ConnImplBenchmark.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java b/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java index 2ebb4e90a..0be22293e 100644 --- a/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java +++ b/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java @@ -52,8 +52,10 @@ public class ConnImplBenchmark { private final String DATASET = "bigquery_test_dataset"; private final String QUERY = "SELECT * FROM bigquery-public-data.new_york_taxi_trips.tlc_yellow_trips_2017 LIMIT %s"; - public static final long NUM_PAGE_ROW_CNT_RATIO = 10; - public static final long NUM_MIN_RESULT_SIZE = 200000; + public static final long NUM_PAGE_ROW_CNT_RATIO = + 10; // ratio of [records in the current page :: total rows] to be met to use read API + public static final long NUM_MIN_RESULT_SIZE = + 200000; // min number of records to use to ReadAPI with @Setup public void setUp() throws IOException { From 15b5fae292f39cf36028bb16a7eb20bad20efc8c Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 27 Apr 2022 10:33:22 +0530 Subject: [PATCH 219/277] updated Warmup iterations to 1 --- .../main/java/com.google.cloud.bigquery/ConnImplBenchmark.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java b/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java index 0be22293e..e15c838d3 100644 --- a/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java +++ b/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java @@ -39,7 +39,7 @@ @Fork(value = 1) @BenchmarkMode(Mode.AverageTime) -@Warmup(iterations = 0) +@Warmup(iterations = 1) @Measurement(iterations = 3) @State(Scope.Benchmark) @OutputTimeUnit(TimeUnit.MILLISECONDS) From 9191dbb5fe843347c3515dca3ed450f9754c8eab Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 27 Apr 2022 11:29:12 +0530 Subject: [PATCH 220/277] updated dataset --- .../main/java/com.google.cloud.bigquery/ConnImplBenchmark.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java b/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java index e15c838d3..36c27eb6a 100644 --- a/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java +++ b/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java @@ -49,7 +49,7 @@ public class ConnImplBenchmark { private ConnectionSettings connectionSettingsReadAPIEnabled, connectionSettingsReadAPIDisabled; private long numBuffRows = 100000L; - private final String DATASET = "bigquery_test_dataset"; + private final String DATASET = "new_york_taxi_trips"; private final String QUERY = "SELECT * FROM bigquery-public-data.new_york_taxi_trips.tlc_yellow_trips_2017 LIMIT %s"; public static final long NUM_PAGE_ROW_CNT_RATIO = From 350767cae36319683131be62082684f0f6fd1eba Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 27 Apr 2022 12:11:08 +0530 Subject: [PATCH 221/277] Fix for testConnectionClose IT --- .../google/cloud/bigquery/ConnectionImpl.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index a27edcf56..6ebaf73fb 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -85,7 +85,7 @@ class ConnectionImpl implements Connection { Executors.newFixedThreadPool(MAX_PROCESS_QUERY_THREADS_CNT); private final Logger logger = Logger.getLogger(this.getClass().getName()); private BigQueryReadClient bqReadClient; - private static final long EXECUTOR_TIMEOUT_SEC = 30; + private static final long EXECUTOR_TIMEOUT_SEC = 5; ConnectionImpl( ConnectionSettings connectionSettings, @@ -116,8 +116,19 @@ class ConnectionImpl implements Connection { @BetaApi @Override public synchronized Boolean close() throws BigQuerySQLException { - return MoreExecutors.shutdownAndAwaitTermination( - queryTaskExecutor, EXECUTOR_TIMEOUT_SEC, TimeUnit.SECONDS); + queryTaskExecutor.shutdownNow(); + try { + queryTaskExecutor.awaitTermination( + EXECUTOR_TIMEOUT_SEC, TimeUnit.SECONDS); // wait for the executor shutdown + } catch (InterruptedException e) { + e.printStackTrace(); + logger.log( + Level.WARNING, + "\n" + Thread.currentThread().getName() + " Exception while awaitTermination", + e); // Logging InterruptedException instead of throwing the exception back, close method + // will return queryTaskExecutor.isShutdown() + } + return queryTaskExecutor.isShutdown(); // check if the executor has been shutdown } /** From b071123c5a1a7e48f9d2488cd8a69804db9e4a82 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Tue, 19 Apr 2022 00:13:29 +0000 Subject: [PATCH 222/277] update labels and close method MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Made Label optional by making it a vararg Made Label optional by making it a vararg Removed null labels Renamed cancel to close Refactored testConnectionCancel Renamed cancel() to close() Renamed testCancel Added testPositionalParams Replaced all the shutdownNows with MoreExecutors.shutdownAndAwaitTermination() --- README.md | 4 +- .../com/google/cloud/bigquery/Connection.java | 20 +++---- .../google/cloud/bigquery/ConnectionImpl.java | 35 ++++++++---- .../cloud/bigquery/ConnectionImplTest.java | 4 +- .../cloud/bigquery/it/ITBigQueryTest.java | 8 +-- .../bigquery/it/ITNightlyBigQueryTest.java | 53 +++++++++++++++++-- 6 files changed, 91 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 0616bfdbd..be04041b2 100644 --- a/README.md +++ b/README.md @@ -59,13 +59,13 @@ implementation 'com.google.cloud:google-cloud-bigquery' If you are using Gradle without BOM, add this to your dependencies ```Groovy -implementation 'com.google.cloud:google-cloud-bigquery:2.10.9' +implementation 'com.google.cloud:google-cloud-bigquery:2.10.10' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.10.9" +libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.10.10" ``` ## Authentication diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java index bf0740ae9..036b52ddc 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java @@ -28,7 +28,7 @@ public interface Connection { /** Sends a query cancel request. This call will return immediately */ @BetaApi - Boolean cancel() throws BigQuerySQLException; + Boolean close() throws BigQuerySQLException; /** * Execute a query dry run that does not return any BigQueryResultSet // TODO: explain more about @@ -75,22 +75,22 @@ public interface Connection { BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException; /** - * Execute a SQL statement with query parameters that returns a single ResultSet + * This method executes a SQL SELECT query * - * @param sql typically a static SQL SELECT statement + * @param sql SQL SELECT query * @param parameters named or positional parameters. The set of query parameters must either be - * all positional or all named parameters. Named parameters are denoted using an @ prefix, - * e.g. @myParam for a parameter named "myParam". + * all positional or all named parameters. * @param labels the labels associated with this query. You can use these to organize and group * your query jobs. Label keys and values can be no longer than 63 characters, can only * contain lowercase letters, numeric characters, underscores and dashes. International - * characters are allowed. Label values are optional. Label keys must start with a letter and - * each label in the list must have a different key. - * @return a ResultSet that contains the data produced by the query - * @exception BigQuerySQLException if a database access error occurs + * characters are allowed. Label values are optional and Label is a Varargs. You should pass + * all the Labels in a single Map .Label keys must start with a letter and each label in the + * list must have a different key. + * @return BigQueryResultSet containing the output of the query + * @throws BigQuerySQLException */ @BetaApi BigQueryResultSet executeSelect( - String sql, List parameters, Map labels) + String sql, List parameters, Map... labels) throws BigQuerySQLException; } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index aebc473c2..a27edcf56 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -44,6 +44,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.util.concurrent.MoreExecutors; import java.io.IOException; import java.util.AbstractList; import java.util.ArrayList; @@ -57,6 +58,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -83,6 +85,7 @@ class ConnectionImpl implements Connection { Executors.newFixedThreadPool(MAX_PROCESS_QUERY_THREADS_CNT); private final Logger logger = Logger.getLogger(this.getClass().getName()); private BigQueryReadClient bqReadClient; + private static final long EXECUTOR_TIMEOUT_SEC = 30; ConnectionImpl( ConnectionSettings connectionSettings, @@ -112,9 +115,9 @@ class ConnectionImpl implements Connection { */ @BetaApi @Override - public synchronized Boolean cancel() throws BigQuerySQLException { - queryTaskExecutor.shutdownNow(); - return queryTaskExecutor.isShutdown(); + public synchronized Boolean close() throws BigQuerySQLException { + return MoreExecutors.shutdownAndAwaitTermination( + queryTaskExecutor, EXECUTOR_TIMEOUT_SEC, TimeUnit.SECONDS); } /** @@ -173,27 +176,33 @@ public BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException { * @param labels the labels associated with this query. You can use these to organize and group * your query jobs. Label keys and values can be no longer than 63 characters, can only * contain lowercase letters, numeric characters, underscores and dashes. International - * characters are allowed. Label values are optional. Label keys must start with a letter and - * each label in the list must have a different key. + * characters are allowed. Label values are optional and Label is a Varargs. You should pass + * all the Labels in a single Map .Label keys must start with a letter and each label in the + * list must have a different key. * @return BigQueryResultSet containing the output of the query * @throws BigQuerySQLException */ @BetaApi @Override public BigQueryResultSet executeSelect( - String sql, List parameters, Map labels) + String sql, List parameters, Map... labels) throws BigQuerySQLException { + Map labelMap = null; + if (labels != null + && labels.length == 1) { // We expect label as a key value pair in a single Map + labelMap = labels[0]; + } try { // use jobs.query if possible if (isFastQuerySupported()) { final String projectId = bigQueryOptions.getProjectId(); final QueryRequest queryRequest = - createQueryRequest(connectionSettings, sql, parameters, labels); + createQueryRequest(connectionSettings, sql, parameters, labelMap); return queryRpc(projectId, queryRequest, parameters != null); } // use jobs.insert otherwise com.google.api.services.bigquery.model.Job queryJob = - createQueryJob(sql, connectionSettings, parameters, labels); + createQueryJob(sql, connectionSettings, parameters, labelMap); JobId jobId = JobId.fromPb(queryJob.getJobReference()); GetQueryResultsResponse firstPage = getQueryResultsFirstPage(jobId); return getResultSet(firstPage, jobId, sql, parameters != null); @@ -514,7 +523,10 @@ void populateBufferAsync( } catch (InterruptedException e) { throw new BigQueryException(0, e.getMessage(), e); } finally { - queryTaskExecutor.shutdownNow(); // Shutdown the thread pool + MoreExecutors.shutdownAndAwaitTermination( + queryTaskExecutor, + EXECUTOR_TIMEOUT_SEC, + TimeUnit.SECONDS); // Shutdown the thread pool } }; @@ -757,7 +769,10 @@ private void processArrowStreamAsync( } finally { try { buffer.put(Tuple.of(null, false)); // marking end of stream - queryTaskExecutor.shutdownNow(); // Shutdown the thread pool + MoreExecutors.shutdownAndAwaitTermination( + queryTaskExecutor, + EXECUTOR_TIMEOUT_SEC, + TimeUnit.SECONDS); // Shutdown the thread pool } catch (InterruptedException e) { logger.log(Level.WARNING, "\n Error occurred ", e); } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java index 06620d8d2..706264683 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java @@ -172,8 +172,8 @@ public void testFastQueryMultiplePages() throws BigQuerySQLException { } @Test - public void testCancel() throws BigQuerySQLException { - boolean cancelled = connection.cancel(); + public void testClose() throws BigQuerySQLException { + boolean cancelled = connection.close(); assertTrue(cancelled); } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index c3757e811..4ce506ef5 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -2617,7 +2617,7 @@ public void testExecuteSelectSinglePageTableRow() throws SQLException { } @Test - public void testConnectionCancel() throws SQLException { + public void testConnectionClose() throws SQLException { String query = "SELECT date, county, state_name, confirmed_cases, deaths FROM " + TABLE_ID_LARGE.getTable() @@ -2634,7 +2634,7 @@ public void testConnectionCancel() throws SQLException { while (rs.next()) { ++cnt; if (cnt > 57000) { // breaking at 57K, query reads 300K - assertTrue(connection.cancel()); // we should be able to cancel the connection + assertTrue(connection.close()); // we should be able to cancel the connection } } assertTrue( @@ -3371,7 +3371,7 @@ public void testExecuteSelectWithPositionalQueryParameters() throws BigQuerySQLE ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); Connection connection = bigquery.createConnection(connectionSettings); List parameters = ImmutableList.of(stringParam, timeStampParam); - BigQueryResultSet rs = connection.executeSelect(query, parameters, null); + BigQueryResultSet rs = connection.executeSelect(query, parameters); assertEquals(2, rs.getTotalRows()); } @@ -3422,7 +3422,7 @@ public void testExecuteSelectWithNamedQueryParameters() throws BigQuerySQLExcept ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); Connection connection = bigquery.createConnection(connectionSettings); List parameters = ImmutableList.of(stringParam, intArrayParam); - BigQueryResultSet rs = connection.executeSelect(query, parameters, null); + BigQueryResultSet rs = connection.executeSelect(query, parameters); assertEquals(2, rs.getTotalRows()); } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java index e5ff18826..b929ad036 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java @@ -36,7 +36,9 @@ import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.InsertAllRequest; import com.google.cloud.bigquery.InsertAllResponse; +import com.google.cloud.bigquery.Parameter; import com.google.cloud.bigquery.QueryJobConfiguration; +import com.google.cloud.bigquery.QueryParameterValue; import com.google.cloud.bigquery.ReadClientConnectionConfiguration; import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.StandardSQLTypeName; @@ -46,6 +48,7 @@ import com.google.cloud.bigquery.TableId; import com.google.cloud.bigquery.TableInfo; import com.google.cloud.bigquery.testing.RemoteBigQueryHelper; +import com.google.common.collect.ImmutableList; import com.google.common.io.BaseEncoding; import java.io.IOException; import java.math.BigDecimal; @@ -89,7 +92,7 @@ public class ITNightlyBigQueryTest { + " JSONField, JSONField.hello, JSONField.id from %s.%s order by IntegerField asc LIMIT %s"; private static final String POSITIONAL_QUERY = String.format( - "select StringField, GeographyField, BooleanField, DateField from %s.%s where DateField = ? LIMIT %s", + "select RecordField.BooleanField, RecordField.StringField, StringField, BooleanField, BytesField, IntegerField, GeographyField, NumericField, BigNumericField, TimeField, DateField, TimestampField, JSONField from %s.%s where DateField = ? and BooleanField = ? and IntegerField > ? and NumericField > ? LIMIT %s", DATASET, TABLE, MULTI_LIMIT_RECS); private static final String QUERY = String.format(BASE_QUERY, DATASET, TABLE, LIMIT_RECS); private static final String MULTI_QUERY = @@ -202,7 +205,7 @@ public void testInvalidQuery() throws BigQuerySQLException { assertNotNull(ex.getMessage()); assertTrue(ex.getMessage().toLowerCase().contains("unexpected keyword into")); } finally { - connection.cancel(); + connection.close(); } } @@ -264,7 +267,7 @@ public void testIterateAndOrder() throws SQLException { ++cnt; } assertEquals(LIMIT_RECS, cnt); // all the records were retrieved - connection.cancel(); + connection.close(); } @Test @@ -311,7 +314,7 @@ public void testMultipleRuns() throws SQLException { } ++cnt; } - connection.cancel(); + connection.close(); totalCnt += cnt; // Repeat the same run connection = getConnection(); @@ -352,11 +355,51 @@ public void testMultipleRuns() throws SQLException { } ++cnt; } - connection.cancel(); + connection.close(); totalCnt += cnt; assertEquals(MULTI_LIMIT_RECS * 2, totalCnt); } + @Test + public void testPositionalParams() + throws SQLException { // Bypasses Read API as it doesnt support Positional Params + Connection connection = getConnection(); + Parameter dateParam = + Parameter.newBuilder() + .setQueryParameterValue(QueryParameterValue.date("2022-01-01")) + .build(); + Parameter boolParam = + Parameter.newBuilder().setQueryParameterValue(QueryParameterValue.bool(true)).build(); + Parameter intParam = + Parameter.newBuilder().setQueryParameterValue(QueryParameterValue.int64(1)).build(); + Parameter numericParam = + Parameter.newBuilder() + .setQueryParameterValue(QueryParameterValue.numeric(new BigDecimal(100))) + .build(); + List parameters = ImmutableList.of(dateParam, boolParam, intParam, numericParam); + + BigQueryResultSet bigQueryResultSet = connection.executeSelect(POSITIONAL_QUERY, parameters); + logger.log(Level.INFO, "Query used: {0}", POSITIONAL_QUERY); + ResultSet rs = bigQueryResultSet.getResultSet(); + int cnt = 0; + while (rs.next()) { + assertFalse(rs.getBoolean("BooleanField")); + assertTrue(0.0d <= rs.getDouble("BigNumericField")); + assertTrue(0 <= rs.getInt("IntegerField")); + assertTrue(0L <= rs.getLong("NumericField")); + assertNotNull(rs.getBytes("BytesField")); + assertNotNull(rs.getTimestamp("TimestampField")); + assertNotNull(rs.getTime("TimeField")); + assertNotNull(rs.getDate("DateField")); + assertNotNull(rs.getString("JSONField")); + assertTrue(rs.getBoolean("BooleanField_1")); + assertNotNull(rs.getString("StringField_1")); + ++cnt; + } + connection.close(); + assertEquals(MULTI_LIMIT_RECS, cnt); + } + // asserts the value of each row private static void testForAllDataTypeValues(ResultSet rs, int cnt) throws SQLException { // Testing JSON type From 9ebf5e7dc789d3393eea33ef644be9f82caf02e4 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 26 Apr 2022 15:18:57 +0530 Subject: [PATCH 223/277] update based on comments update based on feedback Using TimeUnit.DAYS.toMillis for conversion from Days to Millis deleted outdated comment nit update update based on comments updated to using a public dataset updated cancel() to close() using constants for clientConnectionConfiguration using constants for clientConnectionConfiguration updated Warmup iterations to 1 combine commits updated dataset Fix for testConnectionClose IT update based on comments fix lint --- .../ConnImplBenchmark.java | 19 ++-- .../com/google/cloud/bigquery/BigQuery.java | 3 +- ...ueryResultSet.java => BigQueryResult.java} | 4 +- ...ltSetImpl.java => BigQueryResultImpl.java} | 28 +++--- ...SetStats.java => BigQueryResultStats.java} | 2 +- ...Impl.java => BigQueryResultStatsImpl.java} | 4 +- .../com/google/cloud/bigquery/Connection.java | 10 +- .../google/cloud/bigquery/ConnectionImpl.java | 79 ++++++++------- .../cloud/bigquery/ConnectionSettings.java | 2 +- .../com/google/cloud/bigquery/Parameter.java | 4 +- .../ReadClientConnectionConfiguration.java | 2 +- .../cloud/bigquery/ConnectionImplTest.java | 31 +++--- .../cloud/bigquery/it/ITBigQueryTest.java | 95 +++++++++---------- .../bigquery/it/ITNightlyBigQueryTest.java | 34 +++---- 14 files changed, 157 insertions(+), 160 deletions(-) rename google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/{BigQueryResultSet.java => BigQueryResult.java} (92%) rename google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/{BigQueryResultSetImpl.java => BigQueryResultImpl.java} (96%) rename google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/{BigQueryResultSetStats.java => BigQueryResultStats.java} (95%) rename google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/{BigQueryResultSetStatsImpl.java => BigQueryResultStatsImpl.java} (86%) diff --git a/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java b/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java index 8eef2e770..36c27eb6a 100644 --- a/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java +++ b/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java @@ -39,7 +39,7 @@ @Fork(value = 1) @BenchmarkMode(Mode.AverageTime) -@Warmup(iterations = 0) +@Warmup(iterations = 1) @Measurement(iterations = 3) @State(Scope.Benchmark) @OutputTimeUnit(TimeUnit.MILLISECONDS) @@ -49,8 +49,13 @@ public class ConnImplBenchmark { private ConnectionSettings connectionSettingsReadAPIEnabled, connectionSettingsReadAPIDisabled; private long numBuffRows = 100000L; - private final String DATASET = "bigquery_test_dataset"; - private final String QUERY = "SELECT * FROM tlc_yellow_trips_2017_stephwang LIMIT %s"; + private final String DATASET = "new_york_taxi_trips"; + private final String QUERY = + "SELECT * FROM bigquery-public-data.new_york_taxi_trips.tlc_yellow_trips_2017 LIMIT %s"; + public static final long NUM_PAGE_ROW_CNT_RATIO = + 10; // ratio of [records in the current page :: total rows] to be met to use read API + public static final long NUM_MIN_RESULT_SIZE = + 200000; // min number of records to use to ReadAPI with @Setup public void setUp() throws IOException { @@ -59,8 +64,8 @@ public void setUp() throws IOException { clientConnectionConfiguration = ReadClientConnectionConfiguration.newBuilder() - .setTotalToPageRowCountRatio(10L) - .setMinResultSize(200000L) + .setTotalToPageRowCountRatio(NUM_PAGE_ROW_CNT_RATIO) + .setMinResultSize(NUM_MIN_RESULT_SIZE) .setBufferSize(numBuffRows) .build(); @@ -101,7 +106,7 @@ public void iterateRecordsUsingReadAPI(Blackhole blackhole) } catch (Exception e) { e.printStackTrace(); } finally { - connectionReadAPIEnabled.cancel(); // IMP to kill the bg workers + connectionReadAPIEnabled.close(); // IMP to kill the bg workers } blackhole.consume(hash); } @@ -121,7 +126,7 @@ public void iterateRecordsWithoutUsingReadAPI(Blackhole blackhole) } catch (Exception e) { e.printStackTrace(); } finally { - connectionReadAPIDisabled.cancel(); // IMP to kill the bg workers + connectionReadAPIDisabled.close(); // IMP to kill the bg workers } blackhole.consume(hash); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java index e83cb6f18..37288f9c8 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java @@ -761,7 +761,8 @@ public int hashCode() { Job create(JobInfo jobInfo, JobOption... options); /** - * Creates a new BigQuery query connection. + * Creates a new BigQuery query connection used for executing queries (not the same as BigQuery + * connection properties). * *

    Example of creating a query connection. * diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResult.java similarity index 92% rename from google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java rename to google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResult.java index 15fc7ede0..6b0c35f67 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResult.java @@ -18,7 +18,7 @@ import java.sql.ResultSet; -public interface BigQueryResultSet { +public interface BigQueryResult { /** Returns the schema of the results. */ Schema getSchema(); @@ -34,5 +34,5 @@ public interface BigQueryResultSet { ResultSet getResultSet(); /* Returns the query statistics associated with this query. */ - BigQueryResultSetStats getBigQueryResultSetStats(); + BigQueryResultStats getBigQueryResultStats(); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java similarity index 96% rename from google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java rename to google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java index 22307996d..527856be4 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java @@ -29,30 +29,30 @@ import java.util.Map; import java.util.TimeZone; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; import org.apache.arrow.vector.util.JsonStringArrayList; import org.apache.arrow.vector.util.Text; -// TODO: This implementation deals with the JSON response. We can have respective implementations -public class BigQueryResultSetImpl implements BigQueryResultSet { +public class BigQueryResultImpl implements BigQueryResult { private final Schema schema; private final long totalRows; private final BlockingQueue buffer; private T cursor; - private final ResultSetWrapper underlyingResultSet; - private final BigQueryResultSetStats bigQueryResultSetStats; + private final BigQueryResultSet underlyingResultSet; + private final BigQueryResultStats bigQueryResultStats; private final FieldList schemaFieldList; - public BigQueryResultSetImpl( + public BigQueryResultImpl( Schema schema, long totalRows, BlockingQueue buffer, - BigQueryResultSetStats bigQueryResultSetStats) { + BigQueryResultStats bigQueryResultStats) { this.schema = schema; this.totalRows = totalRows; this.buffer = buffer; - this.underlyingResultSet = new ResultSetWrapper(); - this.bigQueryResultSetStats = bigQueryResultSetStats; + this.underlyingResultSet = new BigQueryResultSet(); + this.bigQueryResultStats = bigQueryResultStats; this.schemaFieldList = schema.getFields(); } @@ -71,7 +71,7 @@ public ResultSet getResultSet() { return underlyingResultSet; } - private class ResultSetWrapper extends AbstractJdbcResultSet { + private class BigQueryResultSet extends AbstractJdbcResultSet { @Override /*Advances the result set to the next row, returning false if no such row exists. Potentially blocking operation*/ public boolean next() throws SQLException { @@ -525,9 +525,9 @@ public Date getDate(String fieldName) throws SQLException { } else { Integer dateInt = (Integer) dateObj; long dateInMillis = - Long.valueOf(dateInt) - * (24 * 60 * 60 - * 1000); // For example int 18993 represents 2022-01-01, converting time to + TimeUnit.DAYS.toMillis( + Long.valueOf( + dateInt)); // For example int 18993 represents 2022-01-01, converting time to // milli seconds return new Date(dateInMillis); } @@ -548,7 +548,7 @@ public Date getDate(int columnIndex) throws SQLException { } @Override - public BigQueryResultSetStats getBigQueryResultSetStats() { - return bigQueryResultSetStats; + public BigQueryResultStats getBigQueryResultStats() { + return bigQueryResultStats; } } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetStats.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultStats.java similarity index 95% rename from google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetStats.java rename to google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultStats.java index cf31642f3..aa5ffbc68 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetStats.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultStats.java @@ -18,7 +18,7 @@ import com.google.cloud.bigquery.JobStatistics.SessionInfo; -public interface BigQueryResultSetStats { +public interface BigQueryResultStats { /** Returns detailed statistics for DML statements. */ DmlStats getDmlStats(); diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetStatsImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultStatsImpl.java similarity index 86% rename from google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetStatsImpl.java rename to google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultStatsImpl.java index 31ca6b14a..e0ca84576 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetStatsImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultStatsImpl.java @@ -18,12 +18,12 @@ import com.google.cloud.bigquery.JobStatistics.SessionInfo; -public class BigQueryResultSetStatsImpl implements BigQueryResultSetStats { +public class BigQueryResultStatsImpl implements BigQueryResultStats { private final DmlStats dmlStats; private final SessionInfo sessionInfo; - public BigQueryResultSetStatsImpl(DmlStats dmlStats, SessionInfo sessionInfo) { + public BigQueryResultStatsImpl(DmlStats dmlStats, SessionInfo sessionInfo) { this.dmlStats = dmlStats; this.sessionInfo = sessionInfo; } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java index 036b52ddc..4b9479ba0 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java @@ -31,7 +31,7 @@ public interface Connection { Boolean close() throws BigQuerySQLException; /** - * Execute a query dry run that does not return any BigQueryResultSet // TODO: explain more about + * Execute a query dry run that does not return any BigQueryResult // TODO: explain more about * what this method does here * * @param sql typically a static SQL SELECT statement @@ -56,7 +56,7 @@ public interface Connection { * // .build(); * // Connection connection = bigquery.createConnection(connectionSettings); * String selectQuery = "SELECT corpus FROM `bigquery-public-data.samples.shakespeare` GROUP BY corpus;"; - * try (BigQueryResultSet bqResultSet = connection.executeSelect(selectQuery)) { + * try (BigQueryResult bqResultSet = connection.executeSelect(selectQuery)) { * ResultSet rs = bqResultSet.getResultSet(); * while (rs.next()) { * System.out.printf("%s,", rs.getString("corpus")); @@ -72,7 +72,7 @@ public interface Connection { * @exception BigQuerySQLException if a database access error occurs */ @BetaApi - BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException; + BigQueryResult executeSelect(String sql) throws BigQuerySQLException; /** * This method executes a SQL SELECT query @@ -86,11 +86,11 @@ public interface Connection { * characters are allowed. Label values are optional and Label is a Varargs. You should pass * all the Labels in a single Map .Label keys must start with a letter and each label in the * list must have a different key. - * @return BigQueryResultSet containing the output of the query + * @return BigQueryResult containing the output of the query * @throws BigQuerySQLException */ @BetaApi - BigQueryResultSet executeSelect( + BigQueryResult executeSelect( String sql, List parameters, Map... labels) throws BigQuerySQLException; } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index a27edcf56..e48686d90 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -85,7 +85,7 @@ class ConnectionImpl implements Connection { Executors.newFixedThreadPool(MAX_PROCESS_QUERY_THREADS_CNT); private final Logger logger = Logger.getLogger(this.getClass().getName()); private BigQueryReadClient bqReadClient; - private static final long EXECUTOR_TIMEOUT_SEC = 30; + private static final long EXECUTOR_TIMEOUT_SEC = 5; ConnectionImpl( ConnectionSettings connectionSettings, @@ -116,8 +116,19 @@ class ConnectionImpl implements Connection { @BetaApi @Override public synchronized Boolean close() throws BigQuerySQLException { - return MoreExecutors.shutdownAndAwaitTermination( - queryTaskExecutor, EXECUTOR_TIMEOUT_SEC, TimeUnit.SECONDS); + queryTaskExecutor.shutdownNow(); + try { + queryTaskExecutor.awaitTermination( + EXECUTOR_TIMEOUT_SEC, TimeUnit.SECONDS); // wait for the executor shutdown + } catch (InterruptedException e) { + e.printStackTrace(); + logger.log( + Level.WARNING, + "\n" + Thread.currentThread().getName() + " Exception while awaitTermination", + e); // Logging InterruptedException instead of throwing the exception back, close method + // will return queryTaskExecutor.isShutdown() + } + return queryTaskExecutor.isShutdown(); // check if the executor has been shutdown } /** @@ -143,12 +154,12 @@ public BigQueryDryRunResult dryRun(String sql) throws BigQuerySQLException { * This method executes a SQL SELECT query * * @param sql SQL SELECT statement - * @return BigQueryResultSet containing the output of the query + * @return BigQueryResult containing the output of the query * @throws BigQuerySQLException */ @BetaApi @Override - public BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException { + public BigQueryResult executeSelect(String sql) throws BigQuerySQLException { try { // use jobs.query if all the properties of connectionSettings are supported if (isFastQuerySupported()) { @@ -179,12 +190,12 @@ public BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException { * characters are allowed. Label values are optional and Label is a Varargs. You should pass * all the Labels in a single Map .Label keys must start with a letter and each label in the * list must have a different key. - * @return BigQueryResultSet containing the output of the query + * @return BigQueryResult containing the output of the query * @throws BigQuerySQLException */ @BetaApi @Override - public BigQueryResultSet executeSelect( + public BigQueryResult executeSelect( String sql, List parameters, Map... labels) throws BigQuerySQLException { Map labelMap = null; @@ -212,7 +223,7 @@ public BigQueryResultSet executeSelect( } @VisibleForTesting - BigQueryResultSet getResultSet( + BigQueryResult getResultSet( GetQueryResultsResponse firstPage, JobId jobId, String sql, Boolean hasQueryParameters) { if (firstPage.getJobComplete() && firstPage.getTotalRows() @@ -236,7 +247,7 @@ BigQueryResultSet getResultSet( static class EndOfFieldValueList extends AbstractList< FieldValue> { // A reference of this class is used as a token to inform the thread - // consuming `buffer` BigQueryResultSetImpl that we have run out of records + // consuming `buffer` BigQueryResultImpl that we have run out of records @Override public FieldValue get(int index) { return null; @@ -248,7 +259,7 @@ public int size() { } } - private BigQueryResultSet queryRpc( + private BigQueryResult queryRpc( final String projectId, final QueryRequest queryRequest, Boolean hasQueryParameters) { com.google.api.services.bigquery.model.QueryResponse results; try { @@ -290,26 +301,26 @@ private BigQueryResultSet queryRpc( } @VisibleForTesting - BigQueryResultSetStats getBigQueryResultSetStats(JobId jobId) { + BigQueryResultStats getBigQueryResultSetStats(JobId jobId) { // Create GetQueryResultsResponse query statistics Job queryJob = getQueryJobRpc(jobId); JobStatistics.QueryStatistics statistics = queryJob.getStatistics(); DmlStats dmlStats = statistics.getDmlStats() == null ? null : statistics.getDmlStats(); JobStatistics.SessionInfo sessionInfo = statistics.getSessionInfo() == null ? null : statistics.getSessionInfo(); - return new BigQueryResultSetStatsImpl(dmlStats, sessionInfo); + return new BigQueryResultStatsImpl(dmlStats, sessionInfo); } /* This method processed the first page of GetQueryResultsResponse and then it uses tabledata.list */ @VisibleForTesting - BigQueryResultSet tableDataList(GetQueryResultsResponse firstPage, JobId jobId) { + BigQueryResult tableDataList(GetQueryResultsResponse firstPage, JobId jobId) { Schema schema; long numRows; schema = Schema.fromPb(firstPage.getSchema()); numRows = firstPage.getTotalRows().longValue(); - BigQueryResultSetStats bigQueryResultSetStats = getBigQueryResultSetStats(jobId); + BigQueryResultStats bigQueryResultStats = getBigQueryResultSetStats(jobId); - // Keeps the deserialized records at the row level, which is consumed by BigQueryResultSet + // Keeps the deserialized records at the row level, which is consumed by BigQueryResult BlockingQueue> buffer = new LinkedBlockingDeque<>(bufferSize); // Keeps the parsed FieldValueLists @@ -335,12 +346,12 @@ BigQueryResultSet tableDataList(GetQueryResultsResponse firstPage, JobId jobId) rpcResponseQueue, pageCache, buffer); // spawns a thread to populate the buffer // This will work for pagination as well, as buffer is getting updated asynchronously - return new BigQueryResultSetImpl>( - schema, numRows, buffer, bigQueryResultSetStats); + return new BigQueryResultImpl>( + schema, numRows, buffer, bigQueryResultStats); } @VisibleForTesting - BigQueryResultSet processQueryResponseResults( + BigQueryResult processQueryResponseResults( com.google.api.services.bigquery.model.QueryResponse results) { Schema schema; long numRows; @@ -353,8 +364,7 @@ BigQueryResultSet processQueryResponseResults( results.getSessionInfo() == null ? null : JobStatistics.SessionInfo.fromPb(results.getSessionInfo()); - BigQueryResultSetStats bigQueryResultSetStats = - new BigQueryResultSetStatsImpl(dmlStats, sessionInfo); + BigQueryResultStats bigQueryResultStats = new BigQueryResultStatsImpl(dmlStats, sessionInfo); BlockingQueue> buffer = new LinkedBlockingDeque<>(bufferSize); BlockingQueue, Boolean>> pageCache = @@ -372,8 +382,8 @@ BigQueryResultSet processQueryResponseResults( populateBufferAsync(rpcResponseQueue, pageCache, buffer); - return new BigQueryResultSetImpl>( - schema, numRows, buffer, bigQueryResultSetStats); + return new BigQueryResultImpl>( + schema, numRows, buffer, bigQueryResultStats); } @VisibleForTesting @@ -582,7 +592,7 @@ int getPageCacheSize(Long numBufferedRows, long numRows, Schema schema) { /* Returns query results using either tabledata.list or the high throughput Read API */ @VisibleForTesting - BigQueryResultSet getSubsequentQueryResultsWithJob( + BigQueryResult getSubsequentQueryResultsWithJob( Long totalRows, Long pageRows, JobId jobId, @@ -595,14 +605,14 @@ BigQueryResultSet getSubsequentQueryResultsWithJob( firstPage.getTotalRows().longValue(), Schema.fromPb(firstPage.getSchema()), getBigQueryResultSetStats( - jobId)) // discord first page and stream the entire BigQueryResultSet using + jobId)) // discord first page and stream the entire BigQueryResult using // the Read API : tableDataList(firstPage, jobId); } /* Returns query results using either tabledata.list or the high throughput Read API */ @VisibleForTesting - BigQueryResultSet getSubsequentQueryResultsWithJob( + BigQueryResult getSubsequentQueryResultsWithJob( Long totalRows, Long pageRows, JobId jobId, @@ -619,7 +629,7 @@ BigQueryResultSet getSubsequentQueryResultsWithJob( // any workaround is possible schema, getBigQueryResultSetStats( - jobId)) // discord first page and stream the entire BigQueryResultSet using + jobId)) // discord first page and stream the entire BigQueryResult using // the Read API : tableDataList(firstPage, jobId); } @@ -691,8 +701,8 @@ TableDataList tableDataListRpc(TableId destinationTable, String pageToken) { } @VisibleForTesting - BigQueryResultSet highThroughPutRead( - TableId destinationTable, long totalRows, Schema schema, BigQueryResultSetStats stats) { + BigQueryResult highThroughPutRead( + TableId destinationTable, long totalRows, Schema schema, BigQueryResultStats stats) { try { if (bqReadClient == null) { // if the read client isn't already initialized. Not thread safe. @@ -731,7 +741,7 @@ BigQueryResultSet highThroughPutRead( schema); logger.log(Level.INFO, "\n Using BigQuery Read API"); - return new BigQueryResultSetImpl, Boolean>>( + return new BigQueryResultImpl, Boolean>>( schema, totalRows, buffer, stats); } catch (IOException e) { @@ -1200,8 +1210,8 @@ private com.google.api.services.bigquery.model.Job createDryRunJob(String sql) { private static final Function POSITIONAL_PARAMETER_TO_PB_FUNCTION = value -> { QueryParameter queryParameterPb = new QueryParameter(); - queryParameterPb.setParameterValue(value.getQueryParameterValue().toValuePb()); - queryParameterPb.setParameterType(value.getQueryParameterValue().toTypePb()); + queryParameterPb.setParameterValue(value.getValue().toValuePb()); + queryParameterPb.setParameterType(value.getValue().toTypePb()); return queryParameterPb; }; @@ -1210,8 +1220,8 @@ private com.google.api.services.bigquery.model.Job createDryRunJob(String sql) { value -> { QueryParameter queryParameterPb = new QueryParameter(); queryParameterPb.setName(value.getName()); - queryParameterPb.setParameterValue(value.getQueryParameterValue().toValuePb()); - queryParameterPb.setParameterType(value.getQueryParameterValue().toTypePb()); + queryParameterPb.setParameterValue(value.getValue().toValuePb()); + queryParameterPb.setParameterType(value.getValue().toTypePb()); return queryParameterPb; }; @@ -1220,7 +1230,6 @@ private com.google.api.services.bigquery.model.Job createDryRunJob(String sql) { pb -> Parameter.newBuilder() .setName(pb.getName() == null ? "" : pb.getName()) - .setQueryParameterValue( - QueryParameterValue.fromPb(pb.getParameterValue(), pb.getParameterType())) + .setValue(QueryParameterValue.fromPb(pb.getParameterValue(), pb.getParameterType())) .build(); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java index 2b1821f4a..3a6baa413 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java @@ -282,7 +282,7 @@ public abstract static class Builder { * serialization. * *

    It also sets the maximum number of table rows allowed in buffer before streaming them to - * the BigQueryResultSet. + * the BigQueryResult. * * @param readClientConnectionConfiguration or {@code null} for none */ diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Parameter.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Parameter.java index 0f0796eb9..9959feab9 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Parameter.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Parameter.java @@ -36,7 +36,7 @@ public abstract class Parameter { public abstract String getName(); /** Returns the value for a query parameter along with its type. */ - public abstract QueryParameterValue getQueryParameterValue(); + public abstract QueryParameterValue getValue(); /** Returns a builder pre-populated using the current values of this field. */ public abstract Builder toBuilder(); @@ -62,7 +62,7 @@ public abstract static class Builder { * * @param parameter parameter or {@code null} for none */ - public abstract Builder setQueryParameterValue(QueryParameterValue parameter); + public abstract Builder setValue(QueryParameterValue parameter); /** Creates a {@code Parameter} object. */ public abstract Parameter build(); diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ReadClientConnectionConfiguration.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ReadClientConnectionConfiguration.java index 483ec50e1..e0805a11e 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ReadClientConnectionConfiguration.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ReadClientConnectionConfiguration.java @@ -43,7 +43,7 @@ public abstract static class Builder { /** * Sets the maximum number of table rows allowed in buffer before streaming them to the - * BigQueryResultSet. + * BigQueryResult. */ @Nullable public abstract Builder setBufferSize(Long bufferSize); diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java index 706264683..9af8ce31d 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java @@ -75,11 +75,11 @@ public class ConnectionImplTest { .setMode(Field.Mode.NULLABLE) .build()); private static final TableSchema FAST_QUERY_TABLESCHEMA = QUERY_SCHEMA.toPb(); - private static final BigQueryResultSet BQ_RS_MOCK_RES = - new BigQueryResultSetImpl(QUERY_SCHEMA, 2, null, null); + private static final BigQueryResult BQ_RS_MOCK_RES = + new BigQueryResultImpl(QUERY_SCHEMA, 2, null, null); - private static final BigQueryResultSet BQ_RS_MOCK_RES_MULTI_PAGE = - new BigQueryResultSetImpl(QUERY_SCHEMA, 4, null, null); + private static final BigQueryResult BQ_RS_MOCK_RES_MULTI_PAGE = + new BigQueryResultImpl(QUERY_SCHEMA, 4, null, null); private static final JobId QUERY_JOB = JobId.of(PROJECT, JOB).setLocation(LOCATION); private static final GetQueryResultsResponse GET_QUERY_RESULTS_RESPONSE = @@ -137,7 +137,7 @@ public void testFastQuerySinglePage() throws BigQuerySQLException { .when(connectionSpy) .processQueryResponseResults(any(QueryResponse.class)); - BigQueryResultSet res = connectionSpy.executeSelect(SQL_QUERY); + BigQueryResult res = connectionSpy.executeSelect(SQL_QUERY); assertEquals(res.getTotalRows(), 2); assertEquals(QUERY_SCHEMA, res.getSchema()); verify(connectionSpy, times(1)) @@ -163,7 +163,7 @@ public void testFastQueryMultiplePages() throws BigQuerySQLException { .processQueryResponseResults( any(com.google.api.services.bigquery.model.QueryResponse.class)); - BigQueryResultSet res = connectionSpy.executeSelect(SQL_QUERY); + BigQueryResult res = connectionSpy.executeSelect(SQL_QUERY); assertEquals(res.getTotalRows(), 4); assertEquals(QUERY_SCHEMA, res.getSchema()); verify(connectionSpy, times(1)) @@ -345,7 +345,7 @@ public void testLegacyQuerySinglePage() throws BigQuerySQLException { any(Boolean.class)); when(bigqueryRpcMock.createJobForQuery(any(com.google.api.services.bigquery.model.Job.class))) .thenReturn(jobResponseMock); // RPC call in createQueryJob - BigQueryResultSet res = connectionSpy.executeSelect(SQL_QUERY); + BigQueryResult res = connectionSpy.executeSelect(SQL_QUERY); assertEquals(res.getTotalRows(), 2); assertEquals(QUERY_SCHEMA, res.getSchema()); verify(bigqueryRpcMock, times(1)) @@ -386,7 +386,7 @@ public void testFastQueryLongRunning() throws SQLException { .setRows(tableRows); when(bigqueryRpcMock.queryRpc(any(String.class), any(QueryRequest.class))) .thenReturn(mockQueryRes); - BigQueryResultSet res = connectionSpy.executeSelect(SQL_QUERY); + BigQueryResult res = connectionSpy.executeSelect(SQL_QUERY); assertEquals(res.getTotalRows(), 2); assertEquals(QUERY_SCHEMA, res.getSchema()); verify(bigqueryRpcMock, times(1)).queryRpc(any(String.class), any(QueryRequest.class)); @@ -416,7 +416,7 @@ public void testLegacyQueryMultiplePages() throws SQLException { .setStatistics(jobStatistics); when(bigqueryRpcMock.createJobForQuery(any(com.google.api.services.bigquery.model.Job.class))) .thenReturn(jobResponseMock); // RPC call in createQueryJob - BigQueryResultSet res = connectionSpy.executeSelect(SQL_QUERY); + BigQueryResult res = connectionSpy.executeSelect(SQL_QUERY); assertEquals(res.getTotalRows(), 2); assertEquals(QUERY_SCHEMA, res.getSchema()); verify(bigqueryRpcMock, times(1)) @@ -451,7 +451,7 @@ public void testExecuteSelectSlow() throws BigQuerySQLException { any(JobId.class), any(String.class), any(Boolean.class)); - BigQueryResultSet res = connectionSpy.executeSelect(SQL_QUERY); + BigQueryResult res = connectionSpy.executeSelect(SQL_QUERY); assertEquals(res.getTotalRows(), 2); assertEquals(QUERY_SCHEMA, res.getSchema()); verify(connectionSpy, times(1)) @@ -490,7 +490,7 @@ public void testExecuteSelectSlowWithParams() throws BigQuerySQLException { any(JobId.class), any(String.class), any(Boolean.class)); - BigQueryResultSet res = connectionSpy.executeSelect(SQL_QUERY, parameters, labels); + BigQueryResult res = connectionSpy.executeSelect(SQL_QUERY, parameters, labels); assertEquals(res.getTotalRows(), 2); assertEquals(QUERY_SCHEMA, res.getSchema()); verify(connectionSpy, times(1)) @@ -505,21 +505,18 @@ public void testExecuteSelectSlowWithParams() throws BigQuerySQLException { public void testGetSubsequentQueryResultsWithJob() { ConnectionImpl connectionSpy = Mockito.spy(connection); JobId jobId = mock(JobId.class); - BigQueryResultSetStats bqRsStats = mock(BigQueryResultSetStats.class); + BigQueryResultStats bqRsStats = mock(BigQueryResultStats.class); doReturn(true) .when(connectionSpy) .useReadAPI(any(Long.class), any(Long.class), any(Schema.class), any(Boolean.class)); doReturn(BQ_RS_MOCK_RES) .when(connectionSpy) .highThroughPutRead( - any(TableId.class), - any(Long.class), - any(Schema.class), - any(BigQueryResultSetStats.class)); + any(TableId.class), any(Long.class), any(Schema.class), any(BigQueryResultStats.class)); doReturn(TABLE_NAME).when(connectionSpy).getDestinationTable(any(JobId.class)); doReturn(bqRsStats).when(connectionSpy).getBigQueryResultSetStats(any(JobId.class)); - BigQueryResultSet res = + BigQueryResult res = connectionSpy.getSubsequentQueryResultsWithJob( 10000L, 100L, jobId, GET_QUERY_RESULTS_RESPONSE, false); assertEquals(res.getTotalRows(), 2); diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 4ce506ef5..e4e305336 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -52,7 +52,7 @@ import com.google.cloud.bigquery.BigQueryDryRunResult; import com.google.cloud.bigquery.BigQueryError; import com.google.cloud.bigquery.BigQueryException; -import com.google.cloud.bigquery.BigQueryResultSet; +import com.google.cloud.bigquery.BigQueryResult; import com.google.cloud.bigquery.BigQuerySQLException; import com.google.cloud.bigquery.Clustering; import com.google.cloud.bigquery.Connection; @@ -2494,13 +2494,12 @@ public void testConnectionImplDryRun() throws SQLException { assertEquals( BQ_RESULTSET_EXPECTED_SCHEMA, bigQueryDryRunResultSet.getSchema()); // match the schema List queryParameters = bigQueryDryRunResultSet.getQueryParameters(); - assertEquals( - StandardSQLTypeName.STRING, queryParameters.get(0).getQueryParameterValue().getType()); + assertEquals(StandardSQLTypeName.STRING, queryParameters.get(0).getValue().getType()); } @Test // This test case test the order of the records, making sure that the result is not jumbled up due - // to the multithreaded BigQueryResultSet implementation + // to the multithreaded BigQueryResult implementation public void testBQResultSetMultiThreadedOrder() throws SQLException { String query = "SELECT date FROM " @@ -2512,8 +2511,8 @@ public void testBQResultSetMultiThreadedOrder() throws SQLException { .setNumBufferedRows(10000L) // page size .build(); Connection connection = bigquery.createConnection(connectionSettings); - BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); - ResultSet rs = bigQueryResultSet.getResultSet(); + BigQueryResult bigQueryResult = connection.executeSelect(query); + ResultSet rs = bigQueryResult.getResultSet(); int cnt = 0; assertTrue(rs.next()); ++cnt; @@ -2542,8 +2541,8 @@ public void testBQResultSetPaginationSlowQuery() throws SQLException { // query route gets executed .build(); Connection connection = bigquery.createConnection(connectionSettings); - BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); - ResultSet rs = bigQueryResultSet.getResultSet(); + BigQueryResult bigQueryResult = connection.executeSelect(query); + ResultSet rs = bigQueryResult.getResultSet(); int cnt = 0; while (rs.next()) { // pagination starts after approx 120,000 records assertNotNull(rs.getDate(0)); @@ -2566,12 +2565,12 @@ public void testExecuteSelectSinglePageTableRow() throws SQLException { ConnectionSettings connectionSettings = ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); Connection connection = bigquery.createConnection(connectionSettings); - BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); - ResultSet rs = bigQueryResultSet.getResultSet(); - Schema sc = bigQueryResultSet.getSchema(); + BigQueryResult bigQueryResult = connection.executeSelect(query); + ResultSet rs = bigQueryResult.getResultSet(); + Schema sc = bigQueryResult.getSchema(); assertEquals(BQ_RESULTSET_EXPECTED_SCHEMA, sc); // match the schema - assertEquals(2, bigQueryResultSet.getTotalRows()); // Expecting 2 rows + assertEquals(2, bigQueryResult.getTotalRows()); // Expecting 2 rows assertTrue(rs.next()); // first row // checking for the null or 0 column values @@ -2628,8 +2627,8 @@ public void testConnectionClose() throws SQLException { .setNumBufferedRows(10000L) // page size .build(); Connection connection = bigquery.createConnection(connectionSettings); - BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); - ResultSet rs = bigQueryResultSet.getResultSet(); + BigQueryResult bigQueryResult = connection.executeSelect(query); + ResultSet rs = bigQueryResult.getResultSet(); int cnt = 0; while (rs.next()) { ++cnt; @@ -2655,8 +2654,8 @@ public void testBQResultSetPagination() throws SQLException { .setNumBufferedRows(10000L) // page size .build(); Connection connection = bigquery.createConnection(connectionSettings); - BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); - ResultSet rs = bigQueryResultSet.getResultSet(); + BigQueryResult bigQueryResult = connection.executeSelect(query); + ResultSet rs = bigQueryResult.getResultSet(); int cnt = 0; while (rs.next()) { // pagination starts after approx 120,000 records assertNotNull(rs.getDate(0)); @@ -2684,12 +2683,12 @@ public void testExecuteSelectSinglePageTableRowColInd() throws SQLException { ConnectionSettings connectionSettings = ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); Connection connection = bigquery.createConnection(connectionSettings); - BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); - ResultSet rs = bigQueryResultSet.getResultSet(); - Schema sc = bigQueryResultSet.getSchema(); + BigQueryResult bigQueryResult = connection.executeSelect(query); + ResultSet rs = bigQueryResult.getResultSet(); + Schema sc = bigQueryResult.getSchema(); assertEquals(BQ_RESULTSET_EXPECTED_SCHEMA, sc); // match the schema - assertEquals(2, bigQueryResultSet.getTotalRows()); // Expecting 2 rows + assertEquals(2, bigQueryResult.getTotalRows()); // Expecting 2 rows while (rs.next()) { assertEquals(rs.getString(0), rs.getString("StringField")); assertTrue(rs.getDouble(1) == rs.getDouble("BigNumericField")); @@ -2741,10 +2740,10 @@ public void testExecuteSelectStruct() throws SQLException { ConnectionSettings connectionSettings = ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); Connection connection = bigquery.createConnection(connectionSettings); - BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); - assertEquals(1, bigQueryResultSet.getTotalRows()); + BigQueryResult bigQueryResult = connection.executeSelect(query); + assertEquals(1, bigQueryResult.getTotalRows()); - Schema schema = bigQueryResultSet.getSchema(); + Schema schema = bigQueryResult.getSchema(); assertEquals("address", schema.getFields().get(0).getName()); assertEquals(Field.Mode.NULLABLE, schema.getFields().get(0).getMode()); // Backend is currently returning LegacySQLTypeName. Tracking bug: b/202977620 @@ -2758,7 +2757,7 @@ public void testExecuteSelectStruct() throws SQLException { LegacySQLTypeName.INTEGER, schema.getFields().get(0).getSubFields().get(1).getType()); assertEquals(Field.Mode.NULLABLE, schema.getFields().get(0).getSubFields().get(1).getMode()); - ResultSet rs = bigQueryResultSet.getResultSet(); + ResultSet rs = bigQueryResult.getResultSet(); assertTrue(rs.next()); FieldValueList addressFieldValue = (com.google.cloud.bigquery.FieldValueList) rs.getObject("address"); @@ -2775,10 +2774,10 @@ public void testExecuteSelectStructSubField() throws SQLException { ConnectionSettings connectionSettings = ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); Connection connection = bigquery.createConnection(connectionSettings); - BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); - assertEquals(1, bigQueryResultSet.getTotalRows()); + BigQueryResult bigQueryResult = connection.executeSelect(query); + assertEquals(1, bigQueryResult.getTotalRows()); - Schema schema = bigQueryResultSet.getSchema(); + Schema schema = bigQueryResult.getSchema(); assertEquals("city", schema.getFields().get(0).getName()); assertEquals(Field.Mode.NULLABLE, schema.getFields().get(0).getMode()); // Backend is currently returning LegacySQLTypeName. Tracking bug: b/202977620 @@ -2786,7 +2785,7 @@ public void testExecuteSelectStructSubField() throws SQLException { assertNull( schema.getFields().get(0).getSubFields()); // this is a String field without any subfields - ResultSet rs = bigQueryResultSet.getResultSet(); + ResultSet rs = bigQueryResult.getResultSet(); assertTrue(rs.next()); String cityFieldValue = rs.getString("city"); assertEquals(rs.getString("city"), rs.getObject(0)); @@ -2800,16 +2799,16 @@ public void testExecuteSelectArray() throws SQLException { ConnectionSettings connectionSettings = ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); Connection connection = bigquery.createConnection(connectionSettings); - BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); - assertEquals(1, bigQueryResultSet.getTotalRows()); + BigQueryResult bigQueryResult = connection.executeSelect(query); + assertEquals(1, bigQueryResult.getTotalRows()); - Schema schema = bigQueryResultSet.getSchema(); + Schema schema = bigQueryResult.getSchema(); assertEquals("f0_", schema.getFields().get(0).getName()); assertEquals(Field.Mode.REPEATED, schema.getFields().get(0).getMode()); assertEquals(LegacySQLTypeName.INTEGER, schema.getFields().get(0).getType()); assertNull(schema.getFields().get(0).getSubFields()); // no subfields for Integers - ResultSet rs = bigQueryResultSet.getResultSet(); + ResultSet rs = bigQueryResult.getResultSet(); assertTrue(rs.next()); FieldValueList arrayFieldValue = (com.google.cloud.bigquery.FieldValueList) rs.getObject(0); assertEquals(1, arrayFieldValue.get(0).getLongValue()); @@ -2824,10 +2823,10 @@ public void testExecuteSelectArrayOfStruct() throws SQLException { ConnectionSettings connectionSettings = ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); Connection connection = bigquery.createConnection(connectionSettings); - BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); - assertEquals(1, bigQueryResultSet.getTotalRows()); + BigQueryResult bigQueryResult = connection.executeSelect(query); + assertEquals(1, bigQueryResult.getTotalRows()); - Schema schema = bigQueryResultSet.getSchema(); + Schema schema = bigQueryResult.getSchema(); assertEquals("f0_", schema.getFields().get(0).getName()); assertEquals(Field.Mode.REPEATED, schema.getFields().get(0).getMode()); // Backend is currently returning LegacySQLTypeName. Tracking bug: b/202977620 @@ -2842,7 +2841,7 @@ public void testExecuteSelectArrayOfStruct() throws SQLException { LegacySQLTypeName.INTEGER, schema.getFields().get(0).getSubFields().get(1).getType()); assertEquals(Field.Mode.NULLABLE, schema.getFields().get(0).getSubFields().get(1).getMode()); - ResultSet rs = bigQueryResultSet.getResultSet(); + ResultSet rs = bigQueryResult.getResultSet(); assertTrue(rs.next()); FieldValueList arrayOfStructFieldValue = (com.google.cloud.bigquery.FieldValueList) rs.getObject(0); @@ -3153,9 +3152,8 @@ public void testExecuteSelectSessionSupport() throws BigQuerySQLException { .setCreateSession(true) .build(); Connection connection = bigquery.createConnection(connectionSettings); - BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); - String sessionId = - bigQueryResultSet.getBigQueryResultSetStats().getSessionInfo().getSessionId(); + BigQueryResult bigQueryResult = connection.executeSelect(query); + String sessionId = bigQueryResult.getBigQueryResultStats().getSessionInfo().getSessionId(); assertNotNull(sessionId); } @@ -3363,15 +3361,14 @@ public void testExecuteSelectWithPositionalQueryParameters() throws BigQuerySQLE QueryParameterValue stringParameter = QueryParameterValue.string("stringValue"); QueryParameterValue timestampParameter = QueryParameterValue.timestamp("2014-01-01 07:00:00.000000+00:00"); - Parameter stringParam = Parameter.newBuilder().setQueryParameterValue(stringParameter).build(); - Parameter timeStampParam = - Parameter.newBuilder().setQueryParameterValue(timestampParameter).build(); + Parameter stringParam = Parameter.newBuilder().setValue(stringParameter).build(); + Parameter timeStampParam = Parameter.newBuilder().setValue(timestampParameter).build(); ConnectionSettings connectionSettings = ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); Connection connection = bigquery.createConnection(connectionSettings); List parameters = ImmutableList.of(stringParam, timeStampParam); - BigQueryResultSet rs = connection.executeSelect(query, parameters); + BigQueryResult rs = connection.executeSelect(query, parameters); assertEquals(2, rs.getTotalRows()); } @@ -3408,21 +3405,15 @@ public void testExecuteSelectWithNamedQueryParameters() throws BigQuerySQLExcept QueryParameterValue intArrayParameter = QueryParameterValue.array(new Integer[] {3, 4}, Integer.class); Parameter stringParam = - Parameter.newBuilder() - .setName("stringParam") - .setQueryParameterValue(stringParameter) - .build(); + Parameter.newBuilder().setName("stringParam").setValue(stringParameter).build(); Parameter intArrayParam = - Parameter.newBuilder() - .setName("integerList") - .setQueryParameterValue(intArrayParameter) - .build(); + Parameter.newBuilder().setName("integerList").setValue(intArrayParameter).build(); ConnectionSettings connectionSettings = ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); Connection connection = bigquery.createConnection(connectionSettings); List parameters = ImmutableList.of(stringParam, intArrayParam); - BigQueryResultSet rs = connection.executeSelect(query, parameters); + BigQueryResult rs = connection.executeSelect(query, parameters); assertEquals(2, rs.getTotalRows()); } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java index b929ad036..0e41d4b63 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java @@ -26,7 +26,7 @@ import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryError; import com.google.cloud.bigquery.BigQueryException; -import com.google.cloud.bigquery.BigQueryResultSet; +import com.google.cloud.bigquery.BigQueryResult; import com.google.cloud.bigquery.BigQuerySQLException; import com.google.cloud.bigquery.Connection; import com.google.cloud.bigquery.ConnectionSettings; @@ -199,7 +199,7 @@ public static void afterClass() throws ExecutionException, InterruptedException public void testInvalidQuery() throws BigQuerySQLException { Connection connection = getConnection(); try { - BigQueryResultSet bigQueryResultSet = connection.executeSelect(INVALID_QUERY); + BigQueryResult bigQueryResult = connection.executeSelect(INVALID_QUERY); fail("BigQuerySQLException was expected"); } catch (BigQuerySQLException ex) { assertNotNull(ex.getMessage()); @@ -212,9 +212,9 @@ public void testInvalidQuery() throws BigQuerySQLException { @Test public void testIterateAndOrder() throws SQLException { Connection connection = getConnection(); - BigQueryResultSet bigQueryResultSet = connection.executeSelect(QUERY); + BigQueryResult bigQueryResult = connection.executeSelect(QUERY); logger.log(Level.INFO, "Query used: {0}", QUERY); - ResultSet rs = bigQueryResultSet.getResultSet(); + ResultSet rs = bigQueryResult.getResultSet(); int cnt = 0; int prevIntegerFieldVal = 0; @@ -274,9 +274,9 @@ public void testIterateAndOrder() throws SQLException { public void testMultipleRuns() throws SQLException { Connection connection = getConnection(); - BigQueryResultSet bigQueryResultSet = connection.executeSelect(MULTI_QUERY); + BigQueryResult bigQueryResult = connection.executeSelect(MULTI_QUERY); logger.log(Level.INFO, "Query used: {0}", MULTI_QUERY); - ResultSet rs = bigQueryResultSet.getResultSet(); + ResultSet rs = bigQueryResult.getResultSet(); int cnt = 0; int totalCnt = 0; @@ -318,8 +318,8 @@ public void testMultipleRuns() throws SQLException { totalCnt += cnt; // Repeat the same run connection = getConnection(); - bigQueryResultSet = connection.executeSelect(MULTI_QUERY); - rs = bigQueryResultSet.getResultSet(); + bigQueryResult = connection.executeSelect(MULTI_QUERY); + rs = bigQueryResult.getResultSet(); cnt = 0; prevIntegerFieldVal = 0; while (rs.next()) { @@ -365,22 +365,16 @@ public void testPositionalParams() throws SQLException { // Bypasses Read API as it doesnt support Positional Params Connection connection = getConnection(); Parameter dateParam = - Parameter.newBuilder() - .setQueryParameterValue(QueryParameterValue.date("2022-01-01")) - .build(); - Parameter boolParam = - Parameter.newBuilder().setQueryParameterValue(QueryParameterValue.bool(true)).build(); - Parameter intParam = - Parameter.newBuilder().setQueryParameterValue(QueryParameterValue.int64(1)).build(); + Parameter.newBuilder().setValue(QueryParameterValue.date("2022-01-01")).build(); + Parameter boolParam = Parameter.newBuilder().setValue(QueryParameterValue.bool(true)).build(); + Parameter intParam = Parameter.newBuilder().setValue(QueryParameterValue.int64(1)).build(); Parameter numericParam = - Parameter.newBuilder() - .setQueryParameterValue(QueryParameterValue.numeric(new BigDecimal(100))) - .build(); + Parameter.newBuilder().setValue(QueryParameterValue.numeric(new BigDecimal(100))).build(); List parameters = ImmutableList.of(dateParam, boolParam, intParam, numericParam); - BigQueryResultSet bigQueryResultSet = connection.executeSelect(POSITIONAL_QUERY, parameters); + BigQueryResult bigQueryResult = connection.executeSelect(POSITIONAL_QUERY, parameters); logger.log(Level.INFO, "Query used: {0}", POSITIONAL_QUERY); - ResultSet rs = bigQueryResultSet.getResultSet(); + ResultSet rs = bigQueryResult.getResultSet(); int cnt = 0; while (rs.next()) { assertFalse(rs.getBoolean("BooleanField")); From ad75b17a891a941cbb2812a4b32547d0fd90f2ef Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 29 Apr 2022 11:44:28 +0530 Subject: [PATCH 224/277] Refactored code - Replaced Tuple, Boolean> with Row --- .../google/cloud/bigquery/ConnectionImpl.java | 27 ++++++------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index e48686d90..b149598e2 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -44,7 +44,6 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.common.util.concurrent.MoreExecutors; import java.io.IOException; import java.util.AbstractList; import java.util.ArrayList; @@ -533,10 +532,7 @@ void populateBufferAsync( } catch (InterruptedException e) { throw new BigQueryException(0, e.getMessage(), e); } finally { - MoreExecutors.shutdownAndAwaitTermination( - queryTaskExecutor, - EXECUTOR_TIMEOUT_SEC, - TimeUnit.SECONDS); // Shutdown the thread pool + queryTaskExecutor.shutdownNow(); // Shutdown the thread pool } }; @@ -730,8 +726,7 @@ BigQueryResult highThroughPutRead( ; ReadSession readSession = bqReadClient.createReadSession(builder.build()); - BlockingQueue, Boolean>> buffer = - new LinkedBlockingDeque<>(bufferSize); + BlockingQueue buffer = new LinkedBlockingDeque<>(bufferSize); Map arrowNameToIndex = new HashMap<>(); // deserialize and populate the buffer async, so that the client isn't blocked processArrowStreamAsync( @@ -741,8 +736,7 @@ BigQueryResult highThroughPutRead( schema); logger.log(Level.INFO, "\n Using BigQuery Read API"); - return new BigQueryResultImpl, Boolean>>( - schema, totalRows, buffer, stats); + return new BigQueryResultImpl(schema, totalRows, buffer, stats); } catch (IOException e) { throw BigQueryException.translateAndThrow(e); @@ -751,7 +745,7 @@ BigQueryResult highThroughPutRead( private void processArrowStreamAsync( ReadSession readSession, - BlockingQueue, Boolean>> buffer, + BlockingQueue buffer, ArrowRowReader reader, Schema schema) { @@ -778,11 +772,8 @@ private void processArrowStreamAsync( throw BigQueryException.translateAndThrow(e); } finally { try { - buffer.put(Tuple.of(null, false)); // marking end of stream - MoreExecutors.shutdownAndAwaitTermination( - queryTaskExecutor, - EXECUTOR_TIMEOUT_SEC, - TimeUnit.SECONDS); // Shutdown the thread pool + buffer.put(new BigQueryResultImpl.Row(null, true)); // marking end of stream + queryTaskExecutor.shutdownNow(); // Shutdown the thread pool } catch (InterruptedException e) { logger.log(Level.WARNING, "\n Error occurred ", e); } @@ -822,9 +813,7 @@ private ArrowRowReader(ArrowSchema arrowSchema, Map arrowNameTo /** @param batch object returned from the ReadRowsResponse. */ private void processRows( - ArrowRecordBatch batch, - BlockingQueue, Boolean>> buffer, - Schema schema) + ArrowRecordBatch batch, BlockingQueue buffer, Schema schema) throws IOException { // deserialize the values and consume the hash of the values try { org.apache.arrow.vector.ipc.message.ArrowRecordBatch deserializedBatch = @@ -859,7 +848,7 @@ private void processRows( field.getName()); // can be accessed using the index or Vector/column name curRow.put(field.getName(), curFieldVec.getObject(rowNum)); // Added the raw value } - buffer.put(Tuple.of(curRow, true)); + buffer.put(new BigQueryResultImpl.Row(curRow)); } root.clear(); // TODO: make sure to clear the root while implementing the thread // interruption logic (Connection.close method) From 274b4b74b2cef15d4844f24aebca778afe95fe68 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 29 Apr 2022 11:44:58 +0530 Subject: [PATCH 225/277] Added class Row and Refactored code - Replaced Tuple, Boolean> with Row --- .../cloud/bigquery/BigQueryResultImpl.java | 91 ++++++++++++------- 1 file changed, 57 insertions(+), 34 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java index 527856be4..f20cd4433 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java @@ -16,7 +16,6 @@ package com.google.cloud.bigquery; -import com.google.cloud.Tuple; import java.math.BigDecimal; import java.sql.Date; import java.sql.ResultSet; @@ -35,6 +34,30 @@ public class BigQueryResultImpl implements BigQueryResult { + // This class represents a row of records, the columns are represented as a map + // (columnName:columnValue pair) + static class Row { + private Map value; + private boolean isLast; + + public Row(Map value) { + this.value = value; + } + + public Row(Map value, boolean isLast) { + this.value = value; + this.isLast = isLast; + } + + public Map getValue() { + return value; + } + + public boolean isLast() { + return isLast; + } + } + private final Schema schema; private final long totalRows; private final BlockingQueue buffer; @@ -80,9 +103,9 @@ public boolean next() throws SQLException { if (isEndOfStream(cursor)) { // check for end of stream cursor = null; return false; - } else if (cursor instanceof Tuple) { - Tuple, Boolean> curTup = (Tuple, Boolean>) cursor; - if (!curTup.y()) { // last Tuple + } else if (cursor instanceof Row) { + Row curTup = (Row) cursor; + if (curTup.isLast()) { // last Tuple cursor = null; return false; } @@ -109,11 +132,11 @@ public Object getObject(String fieldName) throws SQLException { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); return fieldValue.getValue() == null ? null : fieldValue.getValue(); } else { // Data received from Read API (Arrow) - Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; - if (!curTuple.x().containsKey(fieldName)) { + Row curRow = (Row) cursor; + if (!curRow.getValue().containsKey(fieldName)) { throw new SQLException(String.format("Field %s not found", fieldName)); } - return curTuple.x().get(fieldName); + return curRow.getValue().get(fieldName); } } @@ -148,11 +171,11 @@ public String getString(String fieldName) throws SQLException { return fieldValue.getStringValue(); } } else { // Data received from Read API (Arrow) - Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; - if (!curTuple.x().containsKey(fieldName)) { + Row curRow = (Row) cursor; + if (!curRow.getValue().containsKey(fieldName)) { throw new SQLException(String.format("Field %s not found", fieldName)); } - Object currentVal = curTuple.x().get(fieldName); + Object currentVal = curRow.getValue().get(fieldName); if (currentVal == null) { return null; } else if (currentVal instanceof JsonStringArrayList) { // arrays @@ -193,11 +216,11 @@ public int getInt(String fieldName) throws SQLException { return fieldValue.getValue() == null ? 0 : fieldValue.getNumericValue().intValue(); } else { // Data received from Read API (Arrow) - Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; - if (!curTuple.x().containsKey(fieldName)) { + Row curRow = (Row) cursor; + if (!curRow.getValue().containsKey(fieldName)) { throw new SQLException(String.format("Field %s not found", fieldName)); } - Object curVal = curTuple.x().get(fieldName); + Object curVal = curRow.getValue().get(fieldName); if (curVal == null) { return 0; } @@ -233,11 +256,11 @@ public long getLong(String fieldName) throws SQLException { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); return fieldValue.getValue() == null ? 0L : fieldValue.getNumericValue().longValue(); } else { // Data received from Read API (Arrow) - Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; - if (!curTuple.x().containsKey(fieldName)) { + Row curRow = (Row) cursor; + if (!curRow.getValue().containsKey(fieldName)) { throw new SQLException(String.format("Field %s not found", fieldName)); } - Object curVal = curTuple.x().get(fieldName); + Object curVal = curRow.getValue().get(fieldName); if (curVal == null) { return 0L; } else if (curVal instanceof Long) { @@ -273,11 +296,11 @@ public double getDouble(String fieldName) throws SQLException { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); return fieldValue.getValue() == null ? 0d : fieldValue.getNumericValue().doubleValue(); } else { // Data received from Read API (Arrow) - Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; - if (!curTuple.x().containsKey(fieldName)) { + Row curRow = (Row) cursor; + if (!curRow.getValue().containsKey(fieldName)) { throw new SQLException(String.format("Field %s not found", fieldName)); } - Object curVal = curTuple.x().get(fieldName); + Object curVal = curRow.getValue().get(fieldName); return curVal == null ? 0.0d : ((BigDecimal) curVal).doubleValue(); } } @@ -341,11 +364,11 @@ public boolean getBoolean(String fieldName) throws SQLException { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); return fieldValue.getValue() != null && fieldValue.getBooleanValue(); } else { // Data received from Read API (Arrow) - Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; - if (!curTuple.x().containsKey(fieldName)) { + Row curRow = (Row) cursor; + if (!curRow.getValue().containsKey(fieldName)) { throw new SQLException(String.format("Field %s not found", fieldName)); } - Object curVal = curTuple.x().get(fieldName); + Object curVal = curRow.getValue().get(fieldName); return curVal != null && (Boolean) curVal; } } @@ -373,11 +396,11 @@ public byte[] getBytes(String fieldName) throws SQLException { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); return fieldValue.getValue() == null ? null : fieldValue.getBytesValue(); } else { // Data received from Read API (Arrow) - Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; - if (!curTuple.x().containsKey(fieldName)) { + Row curRow = (Row) cursor; + if (!curRow.getValue().containsKey(fieldName)) { throw new SQLException(String.format("Field %s not found", fieldName)); } - Object curVal = curTuple.x().get(fieldName); + Object curVal = curRow.getValue().get(fieldName); return curVal == null ? null : (byte[]) curVal; } } @@ -410,11 +433,11 @@ public Timestamp getTimestamp(String fieldName) throws SQLException { / 1000); // getTimestampValue returns time in microseconds, and TimeStamp // expects it in millis } else { - Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; - if (!curTuple.x().containsKey(fieldName)) { + Row curRow = (Row) cursor; + if (!curRow.getValue().containsKey(fieldName)) { throw new SQLException(String.format("Field %s not found", fieldName)); } - Object timeStampVal = curTuple.x().get(fieldName); + Object timeStampVal = curRow.getValue().get(fieldName); return timeStampVal == null ? null : new Timestamp((Long) timeStampVal / 1000); // Timestamp is represented as a Long @@ -449,11 +472,11 @@ public Time getTime(String fieldName) throws SQLException { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); return getTimeFromFieldVal(fieldValue); } else { // Data received from Read API (Arrow) - Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; - if (!curTuple.x().containsKey(fieldName)) { + Row curRow = (Row) cursor; + if (!curRow.getValue().containsKey(fieldName)) { throw new SQLException(String.format("Field %s not found", fieldName)); } - Object timeStampObj = curTuple.x().get(fieldName); + Object timeStampObj = curRow.getValue().get(fieldName); return timeStampObj == null ? null : new Time( @@ -515,11 +538,11 @@ public Date getDate(String fieldName) throws SQLException { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); return fieldValue.getValue() == null ? null : Date.valueOf(fieldValue.getStringValue()); } else { // Data received from Read API (Arrow) - Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; - if (!curTuple.x().containsKey(fieldName)) { + Row curRow = (Row) cursor; + if (!curRow.getValue().containsKey(fieldName)) { throw new SQLException(String.format("Field %s not found", fieldName)); } - Object dateObj = curTuple.x().get(fieldName); + Object dateObj = curRow.getValue().get(fieldName); if (dateObj == null) { return null; } else { From dfa5fd6c19705cdded895adb4aa180a56a6de046 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 29 Apr 2022 17:44:04 +0530 Subject: [PATCH 226/277] Added overloaded constructor with just message as an argument --- .../com/google/cloud/bigquery/BigQuerySQLException.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuerySQLException.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuerySQLException.java index 08c6aff54..672c6ad3f 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuerySQLException.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuerySQLException.java @@ -34,6 +34,12 @@ public BigQuerySQLException() { this.errors = null; } + public BigQuerySQLException( + String msg) { // overloaded constructor with just message as an argument + super(msg); + this.errors = null; + } + public BigQuerySQLException(List errors) { this.errors = errors; } From 69014c43c13e5a3635a674a0487d40a4334ca192 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 29 Apr 2022 17:45:49 +0530 Subject: [PATCH 227/277] refactored and added BigQuerySQLException in next() --- .../cloud/bigquery/BigQueryResultImpl.java | 50 +++++++++++-------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java index f20cd4433..386e81f3e 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java @@ -56,6 +56,14 @@ public Map getValue() { public boolean isLast() { return isLast; } + + public boolean hasField(String fieldName) { + return this.value.containsKey(fieldName); + } + + public Object get(String fieldName) { + return this.value.get(fieldName); + } } private final Schema schema; @@ -109,6 +117,8 @@ public boolean next() throws SQLException { cursor = null; return false; } + } else { // this case should never occur as the cursor will either be a Row of EoS + throw new BigQuerySQLException("Could not process the current row"); } } catch (InterruptedException e) { throw new SQLException("Error occurred while reading buffer"); @@ -133,10 +143,10 @@ public Object getObject(String fieldName) throws SQLException { return fieldValue.getValue() == null ? null : fieldValue.getValue(); } else { // Data received from Read API (Arrow) Row curRow = (Row) cursor; - if (!curRow.getValue().containsKey(fieldName)) { + if (!curRow.hasField(fieldName)) { throw new SQLException(String.format("Field %s not found", fieldName)); } - return curRow.getValue().get(fieldName); + return curRow.get(fieldName); } } @@ -172,10 +182,10 @@ public String getString(String fieldName) throws SQLException { } } else { // Data received from Read API (Arrow) Row curRow = (Row) cursor; - if (!curRow.getValue().containsKey(fieldName)) { + if (!curRow.hasField(fieldName)) { throw new SQLException(String.format("Field %s not found", fieldName)); } - Object currentVal = curRow.getValue().get(fieldName); + Object currentVal = curRow.get(fieldName); if (currentVal == null) { return null; } else if (currentVal instanceof JsonStringArrayList) { // arrays @@ -217,10 +227,10 @@ public int getInt(String fieldName) throws SQLException { } else { // Data received from Read API (Arrow) Row curRow = (Row) cursor; - if (!curRow.getValue().containsKey(fieldName)) { + if (!curRow.hasField(fieldName)) { throw new SQLException(String.format("Field %s not found", fieldName)); } - Object curVal = curRow.getValue().get(fieldName); + Object curVal = curRow.get(fieldName); if (curVal == null) { return 0; } @@ -257,10 +267,10 @@ public long getLong(String fieldName) throws SQLException { return fieldValue.getValue() == null ? 0L : fieldValue.getNumericValue().longValue(); } else { // Data received from Read API (Arrow) Row curRow = (Row) cursor; - if (!curRow.getValue().containsKey(fieldName)) { + if (!curRow.hasField(fieldName)) { throw new SQLException(String.format("Field %s not found", fieldName)); } - Object curVal = curRow.getValue().get(fieldName); + Object curVal = curRow.get(fieldName); if (curVal == null) { return 0L; } else if (curVal instanceof Long) { @@ -297,10 +307,10 @@ public double getDouble(String fieldName) throws SQLException { return fieldValue.getValue() == null ? 0d : fieldValue.getNumericValue().doubleValue(); } else { // Data received from Read API (Arrow) Row curRow = (Row) cursor; - if (!curRow.getValue().containsKey(fieldName)) { + if (!curRow.hasField(fieldName)) { throw new SQLException(String.format("Field %s not found", fieldName)); } - Object curVal = curRow.getValue().get(fieldName); + Object curVal = curRow.get(fieldName); return curVal == null ? 0.0d : ((BigDecimal) curVal).doubleValue(); } } @@ -365,10 +375,10 @@ public boolean getBoolean(String fieldName) throws SQLException { return fieldValue.getValue() != null && fieldValue.getBooleanValue(); } else { // Data received from Read API (Arrow) Row curRow = (Row) cursor; - if (!curRow.getValue().containsKey(fieldName)) { + if (!curRow.hasField(fieldName)) { throw new SQLException(String.format("Field %s not found", fieldName)); } - Object curVal = curRow.getValue().get(fieldName); + Object curVal = curRow.get(fieldName); return curVal != null && (Boolean) curVal; } } @@ -397,10 +407,10 @@ public byte[] getBytes(String fieldName) throws SQLException { return fieldValue.getValue() == null ? null : fieldValue.getBytesValue(); } else { // Data received from Read API (Arrow) Row curRow = (Row) cursor; - if (!curRow.getValue().containsKey(fieldName)) { + if (!curRow.hasField(fieldName)) { throw new SQLException(String.format("Field %s not found", fieldName)); } - Object curVal = curRow.getValue().get(fieldName); + Object curVal = curRow.get(fieldName); return curVal == null ? null : (byte[]) curVal; } } @@ -434,10 +444,10 @@ public Timestamp getTimestamp(String fieldName) throws SQLException { // expects it in millis } else { Row curRow = (Row) cursor; - if (!curRow.getValue().containsKey(fieldName)) { + if (!curRow.hasField(fieldName)) { throw new SQLException(String.format("Field %s not found", fieldName)); } - Object timeStampVal = curRow.getValue().get(fieldName); + Object timeStampVal = curRow.get(fieldName); return timeStampVal == null ? null : new Timestamp((Long) timeStampVal / 1000); // Timestamp is represented as a Long @@ -473,10 +483,10 @@ public Time getTime(String fieldName) throws SQLException { return getTimeFromFieldVal(fieldValue); } else { // Data received from Read API (Arrow) Row curRow = (Row) cursor; - if (!curRow.getValue().containsKey(fieldName)) { + if (!curRow.hasField(fieldName)) { throw new SQLException(String.format("Field %s not found", fieldName)); } - Object timeStampObj = curRow.getValue().get(fieldName); + Object timeStampObj = curRow.get(fieldName); return timeStampObj == null ? null : new Time( @@ -539,10 +549,10 @@ public Date getDate(String fieldName) throws SQLException { return fieldValue.getValue() == null ? null : Date.valueOf(fieldValue.getStringValue()); } else { // Data received from Read API (Arrow) Row curRow = (Row) cursor; - if (!curRow.getValue().containsKey(fieldName)) { + if (!curRow.hasField(fieldName)) { throw new SQLException(String.format("Field %s not found", fieldName)); } - Object dateObj = curRow.getValue().get(fieldName); + Object dateObj = curRow.get(fieldName); if (dateObj == null) { return null; } else { From 5d59bb57cbd76cc464eee5f29dd6d74f92267282 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 29 Apr 2022 18:02:47 +0530 Subject: [PATCH 228/277] refactored and added throwing BigQuerySQLException --- .../cloud/bigquery/BigQueryResultImpl.java | 42 +++++++++---------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java index 386e81f3e..038eab7cc 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java @@ -34,6 +34,9 @@ public class BigQueryResultImpl implements BigQueryResult { + private static final String NULL_CURSOR_MSG = + "Error occurred while reading the cursor. This could happen if getters are called after we are done reading all the records"; + // This class represents a row of records, the columns are represented as a map // (columnName:columnValue pair) static class Row { @@ -121,7 +124,8 @@ public boolean next() throws SQLException { throw new BigQuerySQLException("Could not process the current row"); } } catch (InterruptedException e) { - throw new SQLException("Error occurred while reading buffer"); + throw new SQLException( + "Error occurred while advancing the cursor. This could happen when connection is closed while we call the next method"); } return true; @@ -137,7 +141,7 @@ public Object getObject(String fieldName) throws SQLException { throw new SQLException("fieldName can't be null"); } if (cursor == null) { - return null; + throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); return fieldValue.getValue() == null ? null : fieldValue.getValue(); @@ -168,7 +172,7 @@ public String getString(String fieldName) throws SQLException { throw new SQLException("fieldName can't be null"); } if (cursor == null) { - return null; + throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); if (fieldValue.getValue() == null) { @@ -260,8 +264,7 @@ public long getLong(String fieldName) throws SQLException { throw new SQLException("fieldName can't be null"); } if (cursor == null) { - return 0L; // the column value; if the value is SQL NULL, the value returned is 0 as per - // java.sql.ResultSet definition + throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); return fieldValue.getValue() == null ? 0L : fieldValue.getNumericValue().longValue(); @@ -300,8 +303,7 @@ public double getDouble(String fieldName) throws SQLException { throw new SQLException("fieldName can't be null"); } if (cursor == null) { - return 0d; // the column value; if the value is SQL NULL, the value returned is 0 as per - // java.sql.ResultSet definition + throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); return fieldValue.getValue() == null ? 0d : fieldValue.getNumericValue().doubleValue(); @@ -334,9 +336,7 @@ public BigDecimal getBigDecimal(String fieldName) throws SQLException { throw new SQLException("fieldName can't be null"); } if (cursor == null) { - return null; // the column value (full precision); if the value is SQL NULL, the value - // returned is null in the Java programming language. as per - // java.sql.ResultSet definition + throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); return fieldValue.getValue() == null @@ -350,9 +350,7 @@ public BigDecimal getBigDecimal(String fieldName) throws SQLException { @Override public BigDecimal getBigDecimal(int columnIndex) throws SQLException { if (cursor == null) { - return null; // the column value (full precision); if the value is SQL NULL, the value - // returned is null in the Java programming language. as per - // java.sql.ResultSet definition + throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); return fieldValue.getValue() == null @@ -369,7 +367,7 @@ public boolean getBoolean(String fieldName) throws SQLException { throw new SQLException("fieldName can't be null"); } if (cursor == null) { - return false; // if the value is SQL NULL, the value returned is false + throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); return fieldValue.getValue() != null && fieldValue.getBooleanValue(); @@ -386,7 +384,7 @@ public boolean getBoolean(String fieldName) throws SQLException { @Override public boolean getBoolean(int columnIndex) throws SQLException { if (cursor == null) { - return false; // if the value is SQL NULL, the value returned is false + throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); return fieldValue.getValue() != null && fieldValue.getBooleanValue(); @@ -401,7 +399,7 @@ public byte[] getBytes(String fieldName) throws SQLException { throw new SQLException("fieldName can't be null"); } if (cursor == null) { - return null; // if the value is SQL NULL, the value returned is null + throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); return fieldValue.getValue() == null ? null : fieldValue.getBytesValue(); @@ -430,7 +428,7 @@ public byte[] getBytes(int columnIndex) throws SQLException { @Override public Timestamp getTimestamp(String fieldName) throws SQLException { if (fieldName == null) { - throw new SQLException("fieldName can't be null"); + throw new BigQuerySQLException(NULL_CURSOR_MSG); } if (cursor == null) { return null; // if the value is SQL NULL, the value returned is null @@ -457,7 +455,7 @@ public Timestamp getTimestamp(String fieldName) throws SQLException { @Override public Timestamp getTimestamp(int columnIndex) throws SQLException { if (cursor == null) { - return null; // if the value is SQL NULL, the value returned is null + throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); return fieldValue.getValue() == null @@ -474,7 +472,7 @@ public Timestamp getTimestamp(int columnIndex) throws SQLException { @Override public Time getTime(String fieldName) throws SQLException { if (fieldName == null) { - throw new SQLException("fieldName can't be null"); + throw new BigQuerySQLException(NULL_CURSOR_MSG); } if (cursor == null) { return null; // if the value is SQL NULL, the value returned is null @@ -504,7 +502,7 @@ private int getTimeZoneOffset() { @Override public Time getTime(int columnIndex) throws SQLException { if (cursor == null) { - return null; // if the value is SQL NULL, the value returned is null + throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); return getTimeFromFieldVal(fieldValue); @@ -543,7 +541,7 @@ public Date getDate(String fieldName) throws SQLException { throw new SQLException("fieldName can't be null"); } if (cursor == null) { - return null; // if the value is SQL NULL, the value returned is null + throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); return fieldValue.getValue() == null ? null : Date.valueOf(fieldValue.getStringValue()); @@ -570,7 +568,7 @@ public Date getDate(String fieldName) throws SQLException { @Override public Date getDate(int columnIndex) throws SQLException { if (cursor == null) { - return null; // if the value is SQL NULL, the value returned is null + throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); return fieldValue.getValue() == null ? null : Date.valueOf(fieldValue.getStringValue()); From 90a767a436cc74ce96b2069c18ce09db299ca0ae Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 29 Apr 2022 18:18:38 +0530 Subject: [PATCH 229/277] Added fieldvalue null checks --- .../cloud/bigquery/BigQueryResultImpl.java | 58 +++++++++++++------ 1 file changed, 40 insertions(+), 18 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java index 038eab7cc..bb77a999a 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java @@ -144,7 +144,7 @@ public Object getObject(String fieldName) throws SQLException { throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); - return fieldValue.getValue() == null ? null : fieldValue.getValue(); + return (fieldValue == null || fieldValue.getValue() == null) ? null : fieldValue.getValue(); } else { // Data received from Read API (Arrow) Row curRow = (Row) cursor; if (!curRow.hasField(fieldName)) { @@ -160,7 +160,7 @@ public Object getObject(int columnIndex) throws SQLException { return null; } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); - return fieldValue.getValue() == null ? null : fieldValue.getValue(); + return (fieldValue == null || fieldValue.getValue() == null) ? null : fieldValue.getValue(); } else { // Data received from Read API (Arrow) return getObject(schemaFieldList.get(columnIndex).getName()); } @@ -175,7 +175,7 @@ public String getString(String fieldName) throws SQLException { throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); - if (fieldValue.getValue() == null) { + if ((fieldValue == null || fieldValue.getValue() == null)) { return null; } else if (fieldValue .getAttribute() @@ -211,7 +211,9 @@ public String getString(int columnIndex) throws SQLException { return null; } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); - return fieldValue.getValue() == null ? null : fieldValue.getStringValue(); + return (fieldValue == null || fieldValue.getValue() == null) + ? null + : fieldValue.getStringValue(); } else { // Data received from Read API (Arrow) return getString(schemaFieldList.get(columnIndex).getName()); } @@ -227,7 +229,9 @@ public int getInt(String fieldName) throws SQLException { // java.sql.ResultSet definition } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); - return fieldValue.getValue() == null ? 0 : fieldValue.getNumericValue().intValue(); + return (fieldValue == null || fieldValue.getValue() == null) + ? 0 + : fieldValue.getNumericValue().intValue(); } else { // Data received from Read API (Arrow) Row curRow = (Row) cursor; @@ -252,7 +256,9 @@ public int getInt(int columnIndex) throws SQLException { // java.sql.ResultSet definition } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); - return fieldValue.getValue() == null ? 0 : fieldValue.getNumericValue().intValue(); + return (fieldValue == null || fieldValue.getValue() == null) + ? 0 + : fieldValue.getNumericValue().intValue(); } else { // Data received from Read API (Arrow) return getInt(schemaFieldList.get(columnIndex).getName()); } @@ -267,7 +273,9 @@ public long getLong(String fieldName) throws SQLException { throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); - return fieldValue.getValue() == null ? 0L : fieldValue.getNumericValue().longValue(); + return (fieldValue == null || fieldValue.getValue() == null) + ? 0L + : fieldValue.getNumericValue().longValue(); } else { // Data received from Read API (Arrow) Row curRow = (Row) cursor; if (!curRow.hasField(fieldName)) { @@ -291,7 +299,9 @@ public long getLong(int columnIndex) throws SQLException { // java.sql.ResultSet definition } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); - return fieldValue.getValue() == null ? 0L : fieldValue.getNumericValue().longValue(); + return (fieldValue == null || fieldValue.getValue() == null) + ? 0L + : fieldValue.getNumericValue().longValue(); } else { // Data received from Read API (Arrow) return getInt(schemaFieldList.get(columnIndex).getName()); } @@ -306,7 +316,9 @@ public double getDouble(String fieldName) throws SQLException { throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); - return fieldValue.getValue() == null ? 0d : fieldValue.getNumericValue().doubleValue(); + return (fieldValue == null || fieldValue.getValue() == null) + ? 0d + : fieldValue.getNumericValue().doubleValue(); } else { // Data received from Read API (Arrow) Row curRow = (Row) cursor; if (!curRow.hasField(fieldName)) { @@ -324,7 +336,9 @@ public double getDouble(int columnIndex) throws SQLException { // java.sql.ResultSet definition } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); - return fieldValue.getValue() == null ? 0d : fieldValue.getNumericValue().doubleValue(); + return (fieldValue == null || fieldValue.getValue() == null) + ? 0d + : fieldValue.getNumericValue().doubleValue(); } else { // Data received from Read API (Arrow) return getDouble(schemaFieldList.get(columnIndex).getName()); } @@ -339,7 +353,7 @@ public BigDecimal getBigDecimal(String fieldName) throws SQLException { throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); - return fieldValue.getValue() == null + return (fieldValue == null || fieldValue.getValue() == null) ? null : BigDecimal.valueOf(fieldValue.getNumericValue().doubleValue()); } else { // Data received from Read API (Arrow) @@ -353,7 +367,7 @@ public BigDecimal getBigDecimal(int columnIndex) throws SQLException { throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); - return fieldValue.getValue() == null + return (fieldValue == null || fieldValue.getValue() == null) ? null : BigDecimal.valueOf(fieldValue.getNumericValue().doubleValue()); } else { // Data received from Read API (Arrow) @@ -402,7 +416,9 @@ public byte[] getBytes(String fieldName) throws SQLException { throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); - return fieldValue.getValue() == null ? null : fieldValue.getBytesValue(); + return (fieldValue == null || fieldValue.getValue() == null) + ? null + : fieldValue.getBytesValue(); } else { // Data received from Read API (Arrow) Row curRow = (Row) cursor; if (!curRow.hasField(fieldName)) { @@ -419,7 +435,9 @@ public byte[] getBytes(int columnIndex) throws SQLException { return null; // if the value is SQL NULL, the value returned is null } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); - return fieldValue.getValue() == null ? null : fieldValue.getBytesValue(); + return (fieldValue == null || fieldValue.getValue() == null) + ? null + : fieldValue.getBytesValue(); } else { // Data received from Read API (Arrow) return getBytes(schemaFieldList.get(columnIndex).getName()); } @@ -434,7 +452,7 @@ public Timestamp getTimestamp(String fieldName) throws SQLException { return null; // if the value is SQL NULL, the value returned is null } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); - return fieldValue.getValue() == null + return (fieldValue == null || fieldValue.getValue() == null) ? null : new Timestamp( fieldValue.getTimestampValue() @@ -458,7 +476,7 @@ public Timestamp getTimestamp(int columnIndex) throws SQLException { throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); - return fieldValue.getValue() == null + return (fieldValue == null || fieldValue.getValue() == null) ? null : new Timestamp( fieldValue.getTimestampValue() @@ -544,7 +562,9 @@ public Date getDate(String fieldName) throws SQLException { throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); - return fieldValue.getValue() == null ? null : Date.valueOf(fieldValue.getStringValue()); + return (fieldValue == null || fieldValue.getValue() == null) + ? null + : Date.valueOf(fieldValue.getStringValue()); } else { // Data received from Read API (Arrow) Row curRow = (Row) cursor; if (!curRow.hasField(fieldName)) { @@ -571,7 +591,9 @@ public Date getDate(int columnIndex) throws SQLException { throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); - return fieldValue.getValue() == null ? null : Date.valueOf(fieldValue.getStringValue()); + return (fieldValue == null || fieldValue.getValue() == null) + ? null + : Date.valueOf(fieldValue.getStringValue()); } else { // Data received from Read API (Arrow) return getDate(schemaFieldList.get(columnIndex).getName()); } From 409deb9c4256c302a261a91d3bcd7bfc3ada4cdf Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 29 Apr 2022 18:23:04 +0530 Subject: [PATCH 230/277] modified close --- .../src/main/java/com/google/cloud/bigquery/Connection.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java index 4b9479ba0..8e52da9e9 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java @@ -28,7 +28,7 @@ public interface Connection { /** Sends a query cancel request. This call will return immediately */ @BetaApi - Boolean close() throws BigQuerySQLException; + boolean close() throws BigQuerySQLException; /** * Execute a query dry run that does not return any BigQueryResult // TODO: explain more about From b5a08998fb59b9204f8aebac150e022d98133594 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 29 Apr 2022 18:23:31 +0530 Subject: [PATCH 231/277] modified close --- .../src/main/java/com/google/cloud/bigquery/ConnectionImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index b149598e2..0eb92a6e8 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -114,7 +114,7 @@ class ConnectionImpl implements Connection { */ @BetaApi @Override - public synchronized Boolean close() throws BigQuerySQLException { + public synchronized boolean close() throws BigQuerySQLException { queryTaskExecutor.shutdownNow(); try { queryTaskExecutor.awaitTermination( From 7ff528ff9a075f1aa131c97488e68bb23005071a Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 29 Apr 2022 18:25:46 +0530 Subject: [PATCH 232/277] modified getLong --- .../java/com/google/cloud/bigquery/BigQueryResultImpl.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java index bb77a999a..75acdf585 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java @@ -284,10 +284,8 @@ public long getLong(String fieldName) throws SQLException { Object curVal = curRow.get(fieldName); if (curVal == null) { return 0L; - } else if (curVal instanceof Long) { - return ((Long) curVal).longValue(); - } else { - return ((BigDecimal) curVal).longValue(); + } else { // value will be Long or BigDecimal, but are Number + return ((Number) curVal).longValue(); } } } From f52620e1b720d94857d9f58fad843154bf82baa6 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 26 Apr 2022 17:44:57 +0530 Subject: [PATCH 233/277] address feedback deleted outdated comment update based on comments updated to using a public dataset updated cancel() to close() using constants for clientConnectionConfiguration using constants for clientConnectionConfiguration updated Warmup iterations to 1 updated dataset Fix for testConnectionClose IT --- .../ConnImplBenchmark.java | 19 ++++++++++++------- .../com/google/cloud/bigquery/BigQuery.java | 3 ++- .../cloud/bigquery/BigQueryResultSetImpl.java | 1 - .../google/cloud/bigquery/ConnectionImpl.java | 17 ++++++++++++++--- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java b/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java index 8eef2e770..36c27eb6a 100644 --- a/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java +++ b/benchmark/src/main/java/com.google.cloud.bigquery/ConnImplBenchmark.java @@ -39,7 +39,7 @@ @Fork(value = 1) @BenchmarkMode(Mode.AverageTime) -@Warmup(iterations = 0) +@Warmup(iterations = 1) @Measurement(iterations = 3) @State(Scope.Benchmark) @OutputTimeUnit(TimeUnit.MILLISECONDS) @@ -49,8 +49,13 @@ public class ConnImplBenchmark { private ConnectionSettings connectionSettingsReadAPIEnabled, connectionSettingsReadAPIDisabled; private long numBuffRows = 100000L; - private final String DATASET = "bigquery_test_dataset"; - private final String QUERY = "SELECT * FROM tlc_yellow_trips_2017_stephwang LIMIT %s"; + private final String DATASET = "new_york_taxi_trips"; + private final String QUERY = + "SELECT * FROM bigquery-public-data.new_york_taxi_trips.tlc_yellow_trips_2017 LIMIT %s"; + public static final long NUM_PAGE_ROW_CNT_RATIO = + 10; // ratio of [records in the current page :: total rows] to be met to use read API + public static final long NUM_MIN_RESULT_SIZE = + 200000; // min number of records to use to ReadAPI with @Setup public void setUp() throws IOException { @@ -59,8 +64,8 @@ public void setUp() throws IOException { clientConnectionConfiguration = ReadClientConnectionConfiguration.newBuilder() - .setTotalToPageRowCountRatio(10L) - .setMinResultSize(200000L) + .setTotalToPageRowCountRatio(NUM_PAGE_ROW_CNT_RATIO) + .setMinResultSize(NUM_MIN_RESULT_SIZE) .setBufferSize(numBuffRows) .build(); @@ -101,7 +106,7 @@ public void iterateRecordsUsingReadAPI(Blackhole blackhole) } catch (Exception e) { e.printStackTrace(); } finally { - connectionReadAPIEnabled.cancel(); // IMP to kill the bg workers + connectionReadAPIEnabled.close(); // IMP to kill the bg workers } blackhole.consume(hash); } @@ -121,7 +126,7 @@ public void iterateRecordsWithoutUsingReadAPI(Blackhole blackhole) } catch (Exception e) { e.printStackTrace(); } finally { - connectionReadAPIDisabled.cancel(); // IMP to kill the bg workers + connectionReadAPIDisabled.close(); // IMP to kill the bg workers } blackhole.consume(hash); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java index e83cb6f18..37288f9c8 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java @@ -761,7 +761,8 @@ public int hashCode() { Job create(JobInfo jobInfo, JobOption... options); /** - * Creates a new BigQuery query connection. + * Creates a new BigQuery query connection used for executing queries (not the same as BigQuery + * connection properties). * *

    Example of creating a query connection. * diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java index 2176f1c36..c634d843b 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java @@ -33,7 +33,6 @@ import org.apache.arrow.vector.util.JsonStringArrayList; import org.apache.arrow.vector.util.Text; -// TODO: This implementation deals with the JSON response. We can have respective implementations public class BigQueryResultSetImpl implements BigQueryResultSet { private final Schema schema; diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index a27edcf56..6ebaf73fb 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -85,7 +85,7 @@ class ConnectionImpl implements Connection { Executors.newFixedThreadPool(MAX_PROCESS_QUERY_THREADS_CNT); private final Logger logger = Logger.getLogger(this.getClass().getName()); private BigQueryReadClient bqReadClient; - private static final long EXECUTOR_TIMEOUT_SEC = 30; + private static final long EXECUTOR_TIMEOUT_SEC = 5; ConnectionImpl( ConnectionSettings connectionSettings, @@ -116,8 +116,19 @@ class ConnectionImpl implements Connection { @BetaApi @Override public synchronized Boolean close() throws BigQuerySQLException { - return MoreExecutors.shutdownAndAwaitTermination( - queryTaskExecutor, EXECUTOR_TIMEOUT_SEC, TimeUnit.SECONDS); + queryTaskExecutor.shutdownNow(); + try { + queryTaskExecutor.awaitTermination( + EXECUTOR_TIMEOUT_SEC, TimeUnit.SECONDS); // wait for the executor shutdown + } catch (InterruptedException e) { + e.printStackTrace(); + logger.log( + Level.WARNING, + "\n" + Thread.currentThread().getName() + " Exception while awaitTermination", + e); // Logging InterruptedException instead of throwing the exception back, close method + // will return queryTaskExecutor.isShutdown() + } + return queryTaskExecutor.isShutdown(); // check if the executor has been shutdown } /** From d77320db61af77d0888b01f671abad24dab7b5d9 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 26 Apr 2022 15:18:57 +0530 Subject: [PATCH 234/277] update based on comments update based on feedback Using TimeUnit.DAYS.toMillis for conversion from Days to Millis deleted outdated comment nit update update based on comments updated to using a public dataset updated cancel() to close() using constants for clientConnectionConfiguration using constants for clientConnectionConfiguration updated Warmup iterations to 1 combine commits updated dataset Fix for testConnectionClose IT update based on comments fix lint --- ...ueryResultSet.java => BigQueryResult.java} | 4 +- ...ltSetImpl.java => BigQueryResultImpl.java} | 20 ++-- ...SetStats.java => BigQueryResultStats.java} | 2 +- ...Impl.java => BigQueryResultStatsImpl.java} | 4 +- .../com/google/cloud/bigquery/Connection.java | 10 +- .../google/cloud/bigquery/ConnectionImpl.java | 62 ++++++------ .../cloud/bigquery/ConnectionSettings.java | 2 +- .../com/google/cloud/bigquery/Parameter.java | 4 +- .../ReadClientConnectionConfiguration.java | 2 +- .../cloud/bigquery/ConnectionImplTest.java | 31 +++--- .../cloud/bigquery/it/ITBigQueryTest.java | 95 +++++++++---------- .../bigquery/it/ITNightlyBigQueryTest.java | 34 +++---- 12 files changed, 125 insertions(+), 145 deletions(-) rename google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/{BigQueryResultSet.java => BigQueryResult.java} (92%) rename google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/{BigQueryResultSetImpl.java => BigQueryResultImpl.java} (97%) rename google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/{BigQueryResultSetStats.java => BigQueryResultStats.java} (95%) rename google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/{BigQueryResultSetStatsImpl.java => BigQueryResultStatsImpl.java} (86%) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResult.java similarity index 92% rename from google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java rename to google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResult.java index 15fc7ede0..6b0c35f67 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSet.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResult.java @@ -18,7 +18,7 @@ import java.sql.ResultSet; -public interface BigQueryResultSet { +public interface BigQueryResult { /** Returns the schema of the results. */ Schema getSchema(); @@ -34,5 +34,5 @@ public interface BigQueryResultSet { ResultSet getResultSet(); /* Returns the query statistics associated with this query. */ - BigQueryResultSetStats getBigQueryResultSetStats(); + BigQueryResultStats getBigQueryResultStats(); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java similarity index 97% rename from google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java rename to google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java index c634d843b..527856be4 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java @@ -33,26 +33,26 @@ import org.apache.arrow.vector.util.JsonStringArrayList; import org.apache.arrow.vector.util.Text; -public class BigQueryResultSetImpl implements BigQueryResultSet { +public class BigQueryResultImpl implements BigQueryResult { private final Schema schema; private final long totalRows; private final BlockingQueue buffer; private T cursor; - private final ResultSetWrapper underlyingResultSet; - private final BigQueryResultSetStats bigQueryResultSetStats; + private final BigQueryResultSet underlyingResultSet; + private final BigQueryResultStats bigQueryResultStats; private final FieldList schemaFieldList; - public BigQueryResultSetImpl( + public BigQueryResultImpl( Schema schema, long totalRows, BlockingQueue buffer, - BigQueryResultSetStats bigQueryResultSetStats) { + BigQueryResultStats bigQueryResultStats) { this.schema = schema; this.totalRows = totalRows; this.buffer = buffer; - this.underlyingResultSet = new ResultSetWrapper(); - this.bigQueryResultSetStats = bigQueryResultSetStats; + this.underlyingResultSet = new BigQueryResultSet(); + this.bigQueryResultStats = bigQueryResultStats; this.schemaFieldList = schema.getFields(); } @@ -71,7 +71,7 @@ public ResultSet getResultSet() { return underlyingResultSet; } - private class ResultSetWrapper extends AbstractJdbcResultSet { + private class BigQueryResultSet extends AbstractJdbcResultSet { @Override /*Advances the result set to the next row, returning false if no such row exists. Potentially blocking operation*/ public boolean next() throws SQLException { @@ -548,7 +548,7 @@ public Date getDate(int columnIndex) throws SQLException { } @Override - public BigQueryResultSetStats getBigQueryResultSetStats() { - return bigQueryResultSetStats; + public BigQueryResultStats getBigQueryResultStats() { + return bigQueryResultStats; } } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetStats.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultStats.java similarity index 95% rename from google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetStats.java rename to google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultStats.java index cf31642f3..aa5ffbc68 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetStats.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultStats.java @@ -18,7 +18,7 @@ import com.google.cloud.bigquery.JobStatistics.SessionInfo; -public interface BigQueryResultSetStats { +public interface BigQueryResultStats { /** Returns detailed statistics for DML statements. */ DmlStats getDmlStats(); diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetStatsImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultStatsImpl.java similarity index 86% rename from google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetStatsImpl.java rename to google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultStatsImpl.java index 31ca6b14a..e0ca84576 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultSetStatsImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultStatsImpl.java @@ -18,12 +18,12 @@ import com.google.cloud.bigquery.JobStatistics.SessionInfo; -public class BigQueryResultSetStatsImpl implements BigQueryResultSetStats { +public class BigQueryResultStatsImpl implements BigQueryResultStats { private final DmlStats dmlStats; private final SessionInfo sessionInfo; - public BigQueryResultSetStatsImpl(DmlStats dmlStats, SessionInfo sessionInfo) { + public BigQueryResultStatsImpl(DmlStats dmlStats, SessionInfo sessionInfo) { this.dmlStats = dmlStats; this.sessionInfo = sessionInfo; } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java index 036b52ddc..4b9479ba0 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java @@ -31,7 +31,7 @@ public interface Connection { Boolean close() throws BigQuerySQLException; /** - * Execute a query dry run that does not return any BigQueryResultSet // TODO: explain more about + * Execute a query dry run that does not return any BigQueryResult // TODO: explain more about * what this method does here * * @param sql typically a static SQL SELECT statement @@ -56,7 +56,7 @@ public interface Connection { * // .build(); * // Connection connection = bigquery.createConnection(connectionSettings); * String selectQuery = "SELECT corpus FROM `bigquery-public-data.samples.shakespeare` GROUP BY corpus;"; - * try (BigQueryResultSet bqResultSet = connection.executeSelect(selectQuery)) { + * try (BigQueryResult bqResultSet = connection.executeSelect(selectQuery)) { * ResultSet rs = bqResultSet.getResultSet(); * while (rs.next()) { * System.out.printf("%s,", rs.getString("corpus")); @@ -72,7 +72,7 @@ public interface Connection { * @exception BigQuerySQLException if a database access error occurs */ @BetaApi - BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException; + BigQueryResult executeSelect(String sql) throws BigQuerySQLException; /** * This method executes a SQL SELECT query @@ -86,11 +86,11 @@ public interface Connection { * characters are allowed. Label values are optional and Label is a Varargs. You should pass * all the Labels in a single Map .Label keys must start with a letter and each label in the * list must have a different key. - * @return BigQueryResultSet containing the output of the query + * @return BigQueryResult containing the output of the query * @throws BigQuerySQLException */ @BetaApi - BigQueryResultSet executeSelect( + BigQueryResult executeSelect( String sql, List parameters, Map... labels) throws BigQuerySQLException; } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 6ebaf73fb..e48686d90 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -154,12 +154,12 @@ public BigQueryDryRunResult dryRun(String sql) throws BigQuerySQLException { * This method executes a SQL SELECT query * * @param sql SQL SELECT statement - * @return BigQueryResultSet containing the output of the query + * @return BigQueryResult containing the output of the query * @throws BigQuerySQLException */ @BetaApi @Override - public BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException { + public BigQueryResult executeSelect(String sql) throws BigQuerySQLException { try { // use jobs.query if all the properties of connectionSettings are supported if (isFastQuerySupported()) { @@ -190,12 +190,12 @@ public BigQueryResultSet executeSelect(String sql) throws BigQuerySQLException { * characters are allowed. Label values are optional and Label is a Varargs. You should pass * all the Labels in a single Map .Label keys must start with a letter and each label in the * list must have a different key. - * @return BigQueryResultSet containing the output of the query + * @return BigQueryResult containing the output of the query * @throws BigQuerySQLException */ @BetaApi @Override - public BigQueryResultSet executeSelect( + public BigQueryResult executeSelect( String sql, List parameters, Map... labels) throws BigQuerySQLException { Map labelMap = null; @@ -223,7 +223,7 @@ public BigQueryResultSet executeSelect( } @VisibleForTesting - BigQueryResultSet getResultSet( + BigQueryResult getResultSet( GetQueryResultsResponse firstPage, JobId jobId, String sql, Boolean hasQueryParameters) { if (firstPage.getJobComplete() && firstPage.getTotalRows() @@ -247,7 +247,7 @@ BigQueryResultSet getResultSet( static class EndOfFieldValueList extends AbstractList< FieldValue> { // A reference of this class is used as a token to inform the thread - // consuming `buffer` BigQueryResultSetImpl that we have run out of records + // consuming `buffer` BigQueryResultImpl that we have run out of records @Override public FieldValue get(int index) { return null; @@ -259,7 +259,7 @@ public int size() { } } - private BigQueryResultSet queryRpc( + private BigQueryResult queryRpc( final String projectId, final QueryRequest queryRequest, Boolean hasQueryParameters) { com.google.api.services.bigquery.model.QueryResponse results; try { @@ -301,26 +301,26 @@ private BigQueryResultSet queryRpc( } @VisibleForTesting - BigQueryResultSetStats getBigQueryResultSetStats(JobId jobId) { + BigQueryResultStats getBigQueryResultSetStats(JobId jobId) { // Create GetQueryResultsResponse query statistics Job queryJob = getQueryJobRpc(jobId); JobStatistics.QueryStatistics statistics = queryJob.getStatistics(); DmlStats dmlStats = statistics.getDmlStats() == null ? null : statistics.getDmlStats(); JobStatistics.SessionInfo sessionInfo = statistics.getSessionInfo() == null ? null : statistics.getSessionInfo(); - return new BigQueryResultSetStatsImpl(dmlStats, sessionInfo); + return new BigQueryResultStatsImpl(dmlStats, sessionInfo); } /* This method processed the first page of GetQueryResultsResponse and then it uses tabledata.list */ @VisibleForTesting - BigQueryResultSet tableDataList(GetQueryResultsResponse firstPage, JobId jobId) { + BigQueryResult tableDataList(GetQueryResultsResponse firstPage, JobId jobId) { Schema schema; long numRows; schema = Schema.fromPb(firstPage.getSchema()); numRows = firstPage.getTotalRows().longValue(); - BigQueryResultSetStats bigQueryResultSetStats = getBigQueryResultSetStats(jobId); + BigQueryResultStats bigQueryResultStats = getBigQueryResultSetStats(jobId); - // Keeps the deserialized records at the row level, which is consumed by BigQueryResultSet + // Keeps the deserialized records at the row level, which is consumed by BigQueryResult BlockingQueue> buffer = new LinkedBlockingDeque<>(bufferSize); // Keeps the parsed FieldValueLists @@ -346,12 +346,12 @@ BigQueryResultSet tableDataList(GetQueryResultsResponse firstPage, JobId jobId) rpcResponseQueue, pageCache, buffer); // spawns a thread to populate the buffer // This will work for pagination as well, as buffer is getting updated asynchronously - return new BigQueryResultSetImpl>( - schema, numRows, buffer, bigQueryResultSetStats); + return new BigQueryResultImpl>( + schema, numRows, buffer, bigQueryResultStats); } @VisibleForTesting - BigQueryResultSet processQueryResponseResults( + BigQueryResult processQueryResponseResults( com.google.api.services.bigquery.model.QueryResponse results) { Schema schema; long numRows; @@ -364,8 +364,7 @@ BigQueryResultSet processQueryResponseResults( results.getSessionInfo() == null ? null : JobStatistics.SessionInfo.fromPb(results.getSessionInfo()); - BigQueryResultSetStats bigQueryResultSetStats = - new BigQueryResultSetStatsImpl(dmlStats, sessionInfo); + BigQueryResultStats bigQueryResultStats = new BigQueryResultStatsImpl(dmlStats, sessionInfo); BlockingQueue> buffer = new LinkedBlockingDeque<>(bufferSize); BlockingQueue, Boolean>> pageCache = @@ -383,8 +382,8 @@ BigQueryResultSet processQueryResponseResults( populateBufferAsync(rpcResponseQueue, pageCache, buffer); - return new BigQueryResultSetImpl>( - schema, numRows, buffer, bigQueryResultSetStats); + return new BigQueryResultImpl>( + schema, numRows, buffer, bigQueryResultStats); } @VisibleForTesting @@ -593,7 +592,7 @@ int getPageCacheSize(Long numBufferedRows, long numRows, Schema schema) { /* Returns query results using either tabledata.list or the high throughput Read API */ @VisibleForTesting - BigQueryResultSet getSubsequentQueryResultsWithJob( + BigQueryResult getSubsequentQueryResultsWithJob( Long totalRows, Long pageRows, JobId jobId, @@ -606,14 +605,14 @@ BigQueryResultSet getSubsequentQueryResultsWithJob( firstPage.getTotalRows().longValue(), Schema.fromPb(firstPage.getSchema()), getBigQueryResultSetStats( - jobId)) // discord first page and stream the entire BigQueryResultSet using + jobId)) // discord first page and stream the entire BigQueryResult using // the Read API : tableDataList(firstPage, jobId); } /* Returns query results using either tabledata.list or the high throughput Read API */ @VisibleForTesting - BigQueryResultSet getSubsequentQueryResultsWithJob( + BigQueryResult getSubsequentQueryResultsWithJob( Long totalRows, Long pageRows, JobId jobId, @@ -630,7 +629,7 @@ BigQueryResultSet getSubsequentQueryResultsWithJob( // any workaround is possible schema, getBigQueryResultSetStats( - jobId)) // discord first page and stream the entire BigQueryResultSet using + jobId)) // discord first page and stream the entire BigQueryResult using // the Read API : tableDataList(firstPage, jobId); } @@ -702,8 +701,8 @@ TableDataList tableDataListRpc(TableId destinationTable, String pageToken) { } @VisibleForTesting - BigQueryResultSet highThroughPutRead( - TableId destinationTable, long totalRows, Schema schema, BigQueryResultSetStats stats) { + BigQueryResult highThroughPutRead( + TableId destinationTable, long totalRows, Schema schema, BigQueryResultStats stats) { try { if (bqReadClient == null) { // if the read client isn't already initialized. Not thread safe. @@ -742,7 +741,7 @@ BigQueryResultSet highThroughPutRead( schema); logger.log(Level.INFO, "\n Using BigQuery Read API"); - return new BigQueryResultSetImpl, Boolean>>( + return new BigQueryResultImpl, Boolean>>( schema, totalRows, buffer, stats); } catch (IOException e) { @@ -1211,8 +1210,8 @@ private com.google.api.services.bigquery.model.Job createDryRunJob(String sql) { private static final Function POSITIONAL_PARAMETER_TO_PB_FUNCTION = value -> { QueryParameter queryParameterPb = new QueryParameter(); - queryParameterPb.setParameterValue(value.getQueryParameterValue().toValuePb()); - queryParameterPb.setParameterType(value.getQueryParameterValue().toTypePb()); + queryParameterPb.setParameterValue(value.getValue().toValuePb()); + queryParameterPb.setParameterType(value.getValue().toTypePb()); return queryParameterPb; }; @@ -1221,8 +1220,8 @@ private com.google.api.services.bigquery.model.Job createDryRunJob(String sql) { value -> { QueryParameter queryParameterPb = new QueryParameter(); queryParameterPb.setName(value.getName()); - queryParameterPb.setParameterValue(value.getQueryParameterValue().toValuePb()); - queryParameterPb.setParameterType(value.getQueryParameterValue().toTypePb()); + queryParameterPb.setParameterValue(value.getValue().toValuePb()); + queryParameterPb.setParameterType(value.getValue().toTypePb()); return queryParameterPb; }; @@ -1231,7 +1230,6 @@ private com.google.api.services.bigquery.model.Job createDryRunJob(String sql) { pb -> Parameter.newBuilder() .setName(pb.getName() == null ? "" : pb.getName()) - .setQueryParameterValue( - QueryParameterValue.fromPb(pb.getParameterValue(), pb.getParameterType())) + .setValue(QueryParameterValue.fromPb(pb.getParameterValue(), pb.getParameterType())) .build(); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java index 2b1821f4a..3a6baa413 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java @@ -282,7 +282,7 @@ public abstract static class Builder { * serialization. * *

    It also sets the maximum number of table rows allowed in buffer before streaming them to - * the BigQueryResultSet. + * the BigQueryResult. * * @param readClientConnectionConfiguration or {@code null} for none */ diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Parameter.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Parameter.java index 0f0796eb9..9959feab9 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Parameter.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Parameter.java @@ -36,7 +36,7 @@ public abstract class Parameter { public abstract String getName(); /** Returns the value for a query parameter along with its type. */ - public abstract QueryParameterValue getQueryParameterValue(); + public abstract QueryParameterValue getValue(); /** Returns a builder pre-populated using the current values of this field. */ public abstract Builder toBuilder(); @@ -62,7 +62,7 @@ public abstract static class Builder { * * @param parameter parameter or {@code null} for none */ - public abstract Builder setQueryParameterValue(QueryParameterValue parameter); + public abstract Builder setValue(QueryParameterValue parameter); /** Creates a {@code Parameter} object. */ public abstract Parameter build(); diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ReadClientConnectionConfiguration.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ReadClientConnectionConfiguration.java index 483ec50e1..e0805a11e 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ReadClientConnectionConfiguration.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ReadClientConnectionConfiguration.java @@ -43,7 +43,7 @@ public abstract static class Builder { /** * Sets the maximum number of table rows allowed in buffer before streaming them to the - * BigQueryResultSet. + * BigQueryResult. */ @Nullable public abstract Builder setBufferSize(Long bufferSize); diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java index 706264683..9af8ce31d 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java @@ -75,11 +75,11 @@ public class ConnectionImplTest { .setMode(Field.Mode.NULLABLE) .build()); private static final TableSchema FAST_QUERY_TABLESCHEMA = QUERY_SCHEMA.toPb(); - private static final BigQueryResultSet BQ_RS_MOCK_RES = - new BigQueryResultSetImpl(QUERY_SCHEMA, 2, null, null); + private static final BigQueryResult BQ_RS_MOCK_RES = + new BigQueryResultImpl(QUERY_SCHEMA, 2, null, null); - private static final BigQueryResultSet BQ_RS_MOCK_RES_MULTI_PAGE = - new BigQueryResultSetImpl(QUERY_SCHEMA, 4, null, null); + private static final BigQueryResult BQ_RS_MOCK_RES_MULTI_PAGE = + new BigQueryResultImpl(QUERY_SCHEMA, 4, null, null); private static final JobId QUERY_JOB = JobId.of(PROJECT, JOB).setLocation(LOCATION); private static final GetQueryResultsResponse GET_QUERY_RESULTS_RESPONSE = @@ -137,7 +137,7 @@ public void testFastQuerySinglePage() throws BigQuerySQLException { .when(connectionSpy) .processQueryResponseResults(any(QueryResponse.class)); - BigQueryResultSet res = connectionSpy.executeSelect(SQL_QUERY); + BigQueryResult res = connectionSpy.executeSelect(SQL_QUERY); assertEquals(res.getTotalRows(), 2); assertEquals(QUERY_SCHEMA, res.getSchema()); verify(connectionSpy, times(1)) @@ -163,7 +163,7 @@ public void testFastQueryMultiplePages() throws BigQuerySQLException { .processQueryResponseResults( any(com.google.api.services.bigquery.model.QueryResponse.class)); - BigQueryResultSet res = connectionSpy.executeSelect(SQL_QUERY); + BigQueryResult res = connectionSpy.executeSelect(SQL_QUERY); assertEquals(res.getTotalRows(), 4); assertEquals(QUERY_SCHEMA, res.getSchema()); verify(connectionSpy, times(1)) @@ -345,7 +345,7 @@ public void testLegacyQuerySinglePage() throws BigQuerySQLException { any(Boolean.class)); when(bigqueryRpcMock.createJobForQuery(any(com.google.api.services.bigquery.model.Job.class))) .thenReturn(jobResponseMock); // RPC call in createQueryJob - BigQueryResultSet res = connectionSpy.executeSelect(SQL_QUERY); + BigQueryResult res = connectionSpy.executeSelect(SQL_QUERY); assertEquals(res.getTotalRows(), 2); assertEquals(QUERY_SCHEMA, res.getSchema()); verify(bigqueryRpcMock, times(1)) @@ -386,7 +386,7 @@ public void testFastQueryLongRunning() throws SQLException { .setRows(tableRows); when(bigqueryRpcMock.queryRpc(any(String.class), any(QueryRequest.class))) .thenReturn(mockQueryRes); - BigQueryResultSet res = connectionSpy.executeSelect(SQL_QUERY); + BigQueryResult res = connectionSpy.executeSelect(SQL_QUERY); assertEquals(res.getTotalRows(), 2); assertEquals(QUERY_SCHEMA, res.getSchema()); verify(bigqueryRpcMock, times(1)).queryRpc(any(String.class), any(QueryRequest.class)); @@ -416,7 +416,7 @@ public void testLegacyQueryMultiplePages() throws SQLException { .setStatistics(jobStatistics); when(bigqueryRpcMock.createJobForQuery(any(com.google.api.services.bigquery.model.Job.class))) .thenReturn(jobResponseMock); // RPC call in createQueryJob - BigQueryResultSet res = connectionSpy.executeSelect(SQL_QUERY); + BigQueryResult res = connectionSpy.executeSelect(SQL_QUERY); assertEquals(res.getTotalRows(), 2); assertEquals(QUERY_SCHEMA, res.getSchema()); verify(bigqueryRpcMock, times(1)) @@ -451,7 +451,7 @@ public void testExecuteSelectSlow() throws BigQuerySQLException { any(JobId.class), any(String.class), any(Boolean.class)); - BigQueryResultSet res = connectionSpy.executeSelect(SQL_QUERY); + BigQueryResult res = connectionSpy.executeSelect(SQL_QUERY); assertEquals(res.getTotalRows(), 2); assertEquals(QUERY_SCHEMA, res.getSchema()); verify(connectionSpy, times(1)) @@ -490,7 +490,7 @@ public void testExecuteSelectSlowWithParams() throws BigQuerySQLException { any(JobId.class), any(String.class), any(Boolean.class)); - BigQueryResultSet res = connectionSpy.executeSelect(SQL_QUERY, parameters, labels); + BigQueryResult res = connectionSpy.executeSelect(SQL_QUERY, parameters, labels); assertEquals(res.getTotalRows(), 2); assertEquals(QUERY_SCHEMA, res.getSchema()); verify(connectionSpy, times(1)) @@ -505,21 +505,18 @@ public void testExecuteSelectSlowWithParams() throws BigQuerySQLException { public void testGetSubsequentQueryResultsWithJob() { ConnectionImpl connectionSpy = Mockito.spy(connection); JobId jobId = mock(JobId.class); - BigQueryResultSetStats bqRsStats = mock(BigQueryResultSetStats.class); + BigQueryResultStats bqRsStats = mock(BigQueryResultStats.class); doReturn(true) .when(connectionSpy) .useReadAPI(any(Long.class), any(Long.class), any(Schema.class), any(Boolean.class)); doReturn(BQ_RS_MOCK_RES) .when(connectionSpy) .highThroughPutRead( - any(TableId.class), - any(Long.class), - any(Schema.class), - any(BigQueryResultSetStats.class)); + any(TableId.class), any(Long.class), any(Schema.class), any(BigQueryResultStats.class)); doReturn(TABLE_NAME).when(connectionSpy).getDestinationTable(any(JobId.class)); doReturn(bqRsStats).when(connectionSpy).getBigQueryResultSetStats(any(JobId.class)); - BigQueryResultSet res = + BigQueryResult res = connectionSpy.getSubsequentQueryResultsWithJob( 10000L, 100L, jobId, GET_QUERY_RESULTS_RESPONSE, false); assertEquals(res.getTotalRows(), 2); diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 4ce506ef5..e4e305336 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -52,7 +52,7 @@ import com.google.cloud.bigquery.BigQueryDryRunResult; import com.google.cloud.bigquery.BigQueryError; import com.google.cloud.bigquery.BigQueryException; -import com.google.cloud.bigquery.BigQueryResultSet; +import com.google.cloud.bigquery.BigQueryResult; import com.google.cloud.bigquery.BigQuerySQLException; import com.google.cloud.bigquery.Clustering; import com.google.cloud.bigquery.Connection; @@ -2494,13 +2494,12 @@ public void testConnectionImplDryRun() throws SQLException { assertEquals( BQ_RESULTSET_EXPECTED_SCHEMA, bigQueryDryRunResultSet.getSchema()); // match the schema List queryParameters = bigQueryDryRunResultSet.getQueryParameters(); - assertEquals( - StandardSQLTypeName.STRING, queryParameters.get(0).getQueryParameterValue().getType()); + assertEquals(StandardSQLTypeName.STRING, queryParameters.get(0).getValue().getType()); } @Test // This test case test the order of the records, making sure that the result is not jumbled up due - // to the multithreaded BigQueryResultSet implementation + // to the multithreaded BigQueryResult implementation public void testBQResultSetMultiThreadedOrder() throws SQLException { String query = "SELECT date FROM " @@ -2512,8 +2511,8 @@ public void testBQResultSetMultiThreadedOrder() throws SQLException { .setNumBufferedRows(10000L) // page size .build(); Connection connection = bigquery.createConnection(connectionSettings); - BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); - ResultSet rs = bigQueryResultSet.getResultSet(); + BigQueryResult bigQueryResult = connection.executeSelect(query); + ResultSet rs = bigQueryResult.getResultSet(); int cnt = 0; assertTrue(rs.next()); ++cnt; @@ -2542,8 +2541,8 @@ public void testBQResultSetPaginationSlowQuery() throws SQLException { // query route gets executed .build(); Connection connection = bigquery.createConnection(connectionSettings); - BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); - ResultSet rs = bigQueryResultSet.getResultSet(); + BigQueryResult bigQueryResult = connection.executeSelect(query); + ResultSet rs = bigQueryResult.getResultSet(); int cnt = 0; while (rs.next()) { // pagination starts after approx 120,000 records assertNotNull(rs.getDate(0)); @@ -2566,12 +2565,12 @@ public void testExecuteSelectSinglePageTableRow() throws SQLException { ConnectionSettings connectionSettings = ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); Connection connection = bigquery.createConnection(connectionSettings); - BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); - ResultSet rs = bigQueryResultSet.getResultSet(); - Schema sc = bigQueryResultSet.getSchema(); + BigQueryResult bigQueryResult = connection.executeSelect(query); + ResultSet rs = bigQueryResult.getResultSet(); + Schema sc = bigQueryResult.getSchema(); assertEquals(BQ_RESULTSET_EXPECTED_SCHEMA, sc); // match the schema - assertEquals(2, bigQueryResultSet.getTotalRows()); // Expecting 2 rows + assertEquals(2, bigQueryResult.getTotalRows()); // Expecting 2 rows assertTrue(rs.next()); // first row // checking for the null or 0 column values @@ -2628,8 +2627,8 @@ public void testConnectionClose() throws SQLException { .setNumBufferedRows(10000L) // page size .build(); Connection connection = bigquery.createConnection(connectionSettings); - BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); - ResultSet rs = bigQueryResultSet.getResultSet(); + BigQueryResult bigQueryResult = connection.executeSelect(query); + ResultSet rs = bigQueryResult.getResultSet(); int cnt = 0; while (rs.next()) { ++cnt; @@ -2655,8 +2654,8 @@ public void testBQResultSetPagination() throws SQLException { .setNumBufferedRows(10000L) // page size .build(); Connection connection = bigquery.createConnection(connectionSettings); - BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); - ResultSet rs = bigQueryResultSet.getResultSet(); + BigQueryResult bigQueryResult = connection.executeSelect(query); + ResultSet rs = bigQueryResult.getResultSet(); int cnt = 0; while (rs.next()) { // pagination starts after approx 120,000 records assertNotNull(rs.getDate(0)); @@ -2684,12 +2683,12 @@ public void testExecuteSelectSinglePageTableRowColInd() throws SQLException { ConnectionSettings connectionSettings = ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); Connection connection = bigquery.createConnection(connectionSettings); - BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); - ResultSet rs = bigQueryResultSet.getResultSet(); - Schema sc = bigQueryResultSet.getSchema(); + BigQueryResult bigQueryResult = connection.executeSelect(query); + ResultSet rs = bigQueryResult.getResultSet(); + Schema sc = bigQueryResult.getSchema(); assertEquals(BQ_RESULTSET_EXPECTED_SCHEMA, sc); // match the schema - assertEquals(2, bigQueryResultSet.getTotalRows()); // Expecting 2 rows + assertEquals(2, bigQueryResult.getTotalRows()); // Expecting 2 rows while (rs.next()) { assertEquals(rs.getString(0), rs.getString("StringField")); assertTrue(rs.getDouble(1) == rs.getDouble("BigNumericField")); @@ -2741,10 +2740,10 @@ public void testExecuteSelectStruct() throws SQLException { ConnectionSettings connectionSettings = ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); Connection connection = bigquery.createConnection(connectionSettings); - BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); - assertEquals(1, bigQueryResultSet.getTotalRows()); + BigQueryResult bigQueryResult = connection.executeSelect(query); + assertEquals(1, bigQueryResult.getTotalRows()); - Schema schema = bigQueryResultSet.getSchema(); + Schema schema = bigQueryResult.getSchema(); assertEquals("address", schema.getFields().get(0).getName()); assertEquals(Field.Mode.NULLABLE, schema.getFields().get(0).getMode()); // Backend is currently returning LegacySQLTypeName. Tracking bug: b/202977620 @@ -2758,7 +2757,7 @@ public void testExecuteSelectStruct() throws SQLException { LegacySQLTypeName.INTEGER, schema.getFields().get(0).getSubFields().get(1).getType()); assertEquals(Field.Mode.NULLABLE, schema.getFields().get(0).getSubFields().get(1).getMode()); - ResultSet rs = bigQueryResultSet.getResultSet(); + ResultSet rs = bigQueryResult.getResultSet(); assertTrue(rs.next()); FieldValueList addressFieldValue = (com.google.cloud.bigquery.FieldValueList) rs.getObject("address"); @@ -2775,10 +2774,10 @@ public void testExecuteSelectStructSubField() throws SQLException { ConnectionSettings connectionSettings = ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); Connection connection = bigquery.createConnection(connectionSettings); - BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); - assertEquals(1, bigQueryResultSet.getTotalRows()); + BigQueryResult bigQueryResult = connection.executeSelect(query); + assertEquals(1, bigQueryResult.getTotalRows()); - Schema schema = bigQueryResultSet.getSchema(); + Schema schema = bigQueryResult.getSchema(); assertEquals("city", schema.getFields().get(0).getName()); assertEquals(Field.Mode.NULLABLE, schema.getFields().get(0).getMode()); // Backend is currently returning LegacySQLTypeName. Tracking bug: b/202977620 @@ -2786,7 +2785,7 @@ public void testExecuteSelectStructSubField() throws SQLException { assertNull( schema.getFields().get(0).getSubFields()); // this is a String field without any subfields - ResultSet rs = bigQueryResultSet.getResultSet(); + ResultSet rs = bigQueryResult.getResultSet(); assertTrue(rs.next()); String cityFieldValue = rs.getString("city"); assertEquals(rs.getString("city"), rs.getObject(0)); @@ -2800,16 +2799,16 @@ public void testExecuteSelectArray() throws SQLException { ConnectionSettings connectionSettings = ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); Connection connection = bigquery.createConnection(connectionSettings); - BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); - assertEquals(1, bigQueryResultSet.getTotalRows()); + BigQueryResult bigQueryResult = connection.executeSelect(query); + assertEquals(1, bigQueryResult.getTotalRows()); - Schema schema = bigQueryResultSet.getSchema(); + Schema schema = bigQueryResult.getSchema(); assertEquals("f0_", schema.getFields().get(0).getName()); assertEquals(Field.Mode.REPEATED, schema.getFields().get(0).getMode()); assertEquals(LegacySQLTypeName.INTEGER, schema.getFields().get(0).getType()); assertNull(schema.getFields().get(0).getSubFields()); // no subfields for Integers - ResultSet rs = bigQueryResultSet.getResultSet(); + ResultSet rs = bigQueryResult.getResultSet(); assertTrue(rs.next()); FieldValueList arrayFieldValue = (com.google.cloud.bigquery.FieldValueList) rs.getObject(0); assertEquals(1, arrayFieldValue.get(0).getLongValue()); @@ -2824,10 +2823,10 @@ public void testExecuteSelectArrayOfStruct() throws SQLException { ConnectionSettings connectionSettings = ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); Connection connection = bigquery.createConnection(connectionSettings); - BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); - assertEquals(1, bigQueryResultSet.getTotalRows()); + BigQueryResult bigQueryResult = connection.executeSelect(query); + assertEquals(1, bigQueryResult.getTotalRows()); - Schema schema = bigQueryResultSet.getSchema(); + Schema schema = bigQueryResult.getSchema(); assertEquals("f0_", schema.getFields().get(0).getName()); assertEquals(Field.Mode.REPEATED, schema.getFields().get(0).getMode()); // Backend is currently returning LegacySQLTypeName. Tracking bug: b/202977620 @@ -2842,7 +2841,7 @@ public void testExecuteSelectArrayOfStruct() throws SQLException { LegacySQLTypeName.INTEGER, schema.getFields().get(0).getSubFields().get(1).getType()); assertEquals(Field.Mode.NULLABLE, schema.getFields().get(0).getSubFields().get(1).getMode()); - ResultSet rs = bigQueryResultSet.getResultSet(); + ResultSet rs = bigQueryResult.getResultSet(); assertTrue(rs.next()); FieldValueList arrayOfStructFieldValue = (com.google.cloud.bigquery.FieldValueList) rs.getObject(0); @@ -3153,9 +3152,8 @@ public void testExecuteSelectSessionSupport() throws BigQuerySQLException { .setCreateSession(true) .build(); Connection connection = bigquery.createConnection(connectionSettings); - BigQueryResultSet bigQueryResultSet = connection.executeSelect(query); - String sessionId = - bigQueryResultSet.getBigQueryResultSetStats().getSessionInfo().getSessionId(); + BigQueryResult bigQueryResult = connection.executeSelect(query); + String sessionId = bigQueryResult.getBigQueryResultStats().getSessionInfo().getSessionId(); assertNotNull(sessionId); } @@ -3363,15 +3361,14 @@ public void testExecuteSelectWithPositionalQueryParameters() throws BigQuerySQLE QueryParameterValue stringParameter = QueryParameterValue.string("stringValue"); QueryParameterValue timestampParameter = QueryParameterValue.timestamp("2014-01-01 07:00:00.000000+00:00"); - Parameter stringParam = Parameter.newBuilder().setQueryParameterValue(stringParameter).build(); - Parameter timeStampParam = - Parameter.newBuilder().setQueryParameterValue(timestampParameter).build(); + Parameter stringParam = Parameter.newBuilder().setValue(stringParameter).build(); + Parameter timeStampParam = Parameter.newBuilder().setValue(timestampParameter).build(); ConnectionSettings connectionSettings = ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); Connection connection = bigquery.createConnection(connectionSettings); List parameters = ImmutableList.of(stringParam, timeStampParam); - BigQueryResultSet rs = connection.executeSelect(query, parameters); + BigQueryResult rs = connection.executeSelect(query, parameters); assertEquals(2, rs.getTotalRows()); } @@ -3408,21 +3405,15 @@ public void testExecuteSelectWithNamedQueryParameters() throws BigQuerySQLExcept QueryParameterValue intArrayParameter = QueryParameterValue.array(new Integer[] {3, 4}, Integer.class); Parameter stringParam = - Parameter.newBuilder() - .setName("stringParam") - .setQueryParameterValue(stringParameter) - .build(); + Parameter.newBuilder().setName("stringParam").setValue(stringParameter).build(); Parameter intArrayParam = - Parameter.newBuilder() - .setName("integerList") - .setQueryParameterValue(intArrayParameter) - .build(); + Parameter.newBuilder().setName("integerList").setValue(intArrayParameter).build(); ConnectionSettings connectionSettings = ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); Connection connection = bigquery.createConnection(connectionSettings); List parameters = ImmutableList.of(stringParam, intArrayParam); - BigQueryResultSet rs = connection.executeSelect(query, parameters); + BigQueryResult rs = connection.executeSelect(query, parameters); assertEquals(2, rs.getTotalRows()); } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java index b929ad036..0e41d4b63 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java @@ -26,7 +26,7 @@ import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryError; import com.google.cloud.bigquery.BigQueryException; -import com.google.cloud.bigquery.BigQueryResultSet; +import com.google.cloud.bigquery.BigQueryResult; import com.google.cloud.bigquery.BigQuerySQLException; import com.google.cloud.bigquery.Connection; import com.google.cloud.bigquery.ConnectionSettings; @@ -199,7 +199,7 @@ public static void afterClass() throws ExecutionException, InterruptedException public void testInvalidQuery() throws BigQuerySQLException { Connection connection = getConnection(); try { - BigQueryResultSet bigQueryResultSet = connection.executeSelect(INVALID_QUERY); + BigQueryResult bigQueryResult = connection.executeSelect(INVALID_QUERY); fail("BigQuerySQLException was expected"); } catch (BigQuerySQLException ex) { assertNotNull(ex.getMessage()); @@ -212,9 +212,9 @@ public void testInvalidQuery() throws BigQuerySQLException { @Test public void testIterateAndOrder() throws SQLException { Connection connection = getConnection(); - BigQueryResultSet bigQueryResultSet = connection.executeSelect(QUERY); + BigQueryResult bigQueryResult = connection.executeSelect(QUERY); logger.log(Level.INFO, "Query used: {0}", QUERY); - ResultSet rs = bigQueryResultSet.getResultSet(); + ResultSet rs = bigQueryResult.getResultSet(); int cnt = 0; int prevIntegerFieldVal = 0; @@ -274,9 +274,9 @@ public void testIterateAndOrder() throws SQLException { public void testMultipleRuns() throws SQLException { Connection connection = getConnection(); - BigQueryResultSet bigQueryResultSet = connection.executeSelect(MULTI_QUERY); + BigQueryResult bigQueryResult = connection.executeSelect(MULTI_QUERY); logger.log(Level.INFO, "Query used: {0}", MULTI_QUERY); - ResultSet rs = bigQueryResultSet.getResultSet(); + ResultSet rs = bigQueryResult.getResultSet(); int cnt = 0; int totalCnt = 0; @@ -318,8 +318,8 @@ public void testMultipleRuns() throws SQLException { totalCnt += cnt; // Repeat the same run connection = getConnection(); - bigQueryResultSet = connection.executeSelect(MULTI_QUERY); - rs = bigQueryResultSet.getResultSet(); + bigQueryResult = connection.executeSelect(MULTI_QUERY); + rs = bigQueryResult.getResultSet(); cnt = 0; prevIntegerFieldVal = 0; while (rs.next()) { @@ -365,22 +365,16 @@ public void testPositionalParams() throws SQLException { // Bypasses Read API as it doesnt support Positional Params Connection connection = getConnection(); Parameter dateParam = - Parameter.newBuilder() - .setQueryParameterValue(QueryParameterValue.date("2022-01-01")) - .build(); - Parameter boolParam = - Parameter.newBuilder().setQueryParameterValue(QueryParameterValue.bool(true)).build(); - Parameter intParam = - Parameter.newBuilder().setQueryParameterValue(QueryParameterValue.int64(1)).build(); + Parameter.newBuilder().setValue(QueryParameterValue.date("2022-01-01")).build(); + Parameter boolParam = Parameter.newBuilder().setValue(QueryParameterValue.bool(true)).build(); + Parameter intParam = Parameter.newBuilder().setValue(QueryParameterValue.int64(1)).build(); Parameter numericParam = - Parameter.newBuilder() - .setQueryParameterValue(QueryParameterValue.numeric(new BigDecimal(100))) - .build(); + Parameter.newBuilder().setValue(QueryParameterValue.numeric(new BigDecimal(100))).build(); List parameters = ImmutableList.of(dateParam, boolParam, intParam, numericParam); - BigQueryResultSet bigQueryResultSet = connection.executeSelect(POSITIONAL_QUERY, parameters); + BigQueryResult bigQueryResult = connection.executeSelect(POSITIONAL_QUERY, parameters); logger.log(Level.INFO, "Query used: {0}", POSITIONAL_QUERY); - ResultSet rs = bigQueryResultSet.getResultSet(); + ResultSet rs = bigQueryResult.getResultSet(); int cnt = 0; while (rs.next()) { assertFalse(rs.getBoolean("BooleanField")); From 6a7875ea4581383b0324b551577434fe60f76b7f Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Tue, 19 Apr 2022 04:14:40 +0200 Subject: [PATCH 235/277] chore(deps): update dependency com.google.cloud:google-cloud-bigquery to v2.10.10 (#1995) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update dependency com.google.cloud:google-cloud-bigquery to v2.10.10 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot --- README.md | 2 +- samples/install-without-bom/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index be04041b2..cc3ab5f44 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ If you are using Maven without BOM, add this to your dependencies: com.google.cloud google-cloud-bigquery - 2.10.9 + 2.10.10 ``` diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index 24a0de472..9d69b7909 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -45,7 +45,7 @@ com.google.cloud google-cloud-bigquery - 2.10.9 + 2.10.10 From b1746340bcdf2d96d36d1721b11f6abc02aeaced Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Tue, 19 Apr 2022 20:49:54 +0530 Subject: [PATCH 236/277] feat: next release from main branch is 2.10.9 (#1996) --- .github/release-please.yml | 4 ++++ .github/sync-repo-settings.yaml | 15 +++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/.github/release-please.yml b/.github/release-please.yml index 473d7f07c..c87ad8d73 100644 --- a/.github/release-please.yml +++ b/.github/release-please.yml @@ -11,6 +11,10 @@ branches: handleGHRelease: true releaseType: java-backport branch: 2.3.x + - bumpMinorPreMajor: true + handleGHRelease: true + releaseType: java-backport + branch: 2.10.x bumpMinorPreMajor: true handleGHRelease: true releaseType: java-yoshi diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index 960ae2ba0..1bede3ce6 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -62,6 +62,21 @@ branchProtectionRules: - 'Kokoro - Test: Integration' - cla/google - OwlBot Post Processor + - pattern: 2.10.x + isAdminEnforced: true + requiredApprovingReviewCount: 1 + requiresCodeOwnerReviews: true + requiresStrictStatusChecks: false + requiredStatusCheckContexts: + - dependencies (8) + - dependencies (11) + - lint + - clirr + - units (8) + - units (11) + - 'Kokoro - Test: Integration' + - cla/google + - OwlBot Post Processor permissionRules: - team: api-bigquery permission: admin From 849d22b462422abf06fbc680893905e3fb548f26 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Wed, 20 Apr 2022 18:39:26 +0200 Subject: [PATCH 237/277] build(deps): update dependency com.google.cloud:google-cloud-shared-config to v1.3.3 (#2000) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f3c958f18..57953b9a4 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,7 @@ com.google.cloud google-cloud-shared-config - 1.3.2 + 1.3.3 From 91af8255cd5bd4f8370c9aec6c66c6cd13fe5d56 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Wed, 20 Apr 2022 22:31:03 +0200 Subject: [PATCH 238/277] test(deps): update dependency org.mockito:mockito-core to v4.5.0 (#1999) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 57953b9a4..167de04ef 100644 --- a/pom.xml +++ b/pom.xml @@ -151,7 +151,7 @@ org.mockito mockito-core - 4.4.0 + 4.5.0 test From 5689ec05c8b52ff2475d62f797e049e542b59171 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Thu, 21 Apr 2022 16:28:57 +0200 Subject: [PATCH 239/277] test(deps): update dependency org.mockito:mockito-core to v4.5.1 (#2003) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 167de04ef..7bb4ff5f9 100644 --- a/pom.xml +++ b/pom.xml @@ -151,7 +151,7 @@ org.mockito mockito-core - 4.5.0 + 4.5.1 test From 0e4122e29ed8f335cbad5efac010d864f00c694b Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Thu, 21 Apr 2022 16:29:12 +0200 Subject: [PATCH 240/277] build(deps): update dependency org.apache.maven.plugins:maven-javadoc-plugin to v3.4.0 (#2002) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7bb4ff5f9..e466bc612 100644 --- a/pom.xml +++ b/pom.xml @@ -255,7 +255,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.3.2 + 3.4.0 html From 6e9083a9beff632a2d4b7af56ed9d5ff75bd2dec Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Thu, 21 Apr 2022 19:42:31 +0200 Subject: [PATCH 241/277] build(deps): update dependency org.sonatype.plugins:nexus-staging-maven-plugin to v1.6.13 (#2004) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [![WhiteSource Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com) This PR contains the following updates: | Package | Change | Age | Adoption | Passing | Confidence | |---|---|---|---|---|---| | [org.sonatype.plugins:nexus-staging-maven-plugin](http://www.sonatype.com/) ([source](https://togithub.com/sonatype/nexus-maven-plugins)) | `1.6.12` -> `1.6.13` | [![age](https://badges.renovateapi.com/packages/maven/org.sonatype.plugins:nexus-staging-maven-plugin/1.6.13/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/maven/org.sonatype.plugins:nexus-staging-maven-plugin/1.6.13/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/maven/org.sonatype.plugins:nexus-staging-maven-plugin/1.6.13/compatibility-slim/1.6.12)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/maven/org.sonatype.plugins:nexus-staging-maven-plugin/1.6.13/confidence-slim/1.6.12)](https://docs.renovatebot.com/merge-confidence/) | --- ### Release Notes

    sonatype/nexus-maven-plugins ### [`v1.6.13`](https://togithub.com/sonatype/nexus-maven-plugins/compare/release-1.6.12...release-1.6.13) [Compare Source](https://togithub.com/sonatype/nexus-maven-plugins/compare/release-1.6.12...release-1.6.13)
    --- ### Configuration 📅 **Schedule**: At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, click this checkbox. --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#github/googleapis/java-bigquery). --- benchmark/pom.xml | 2 +- samples/pom.xml | 2 +- samples/snippets/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmark/pom.xml b/benchmark/pom.xml index 10f47165e..db892112c 100644 --- a/benchmark/pom.xml +++ b/benchmark/pom.xml @@ -83,7 +83,7 @@ org.sonatype.plugins nexus-staging-maven-plugin - 1.6.12 + 1.6.13 true diff --git a/samples/pom.xml b/samples/pom.xml index 4e280e644..cb1ba7802 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -63,7 +63,7 @@ org.sonatype.plugins nexus-staging-maven-plugin - 1.6.12 + 1.6.13 true diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index 25edf810d..ff98571aa 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -117,7 +117,7 @@ org.sonatype.plugins nexus-staging-maven-plugin - 1.6.12 + 1.6.13 true From 17bbe83331d7126dd343d02533df5101188b2552 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 22 Apr 2022 16:12:21 +0200 Subject: [PATCH 242/277] chore(deps): update dependency com.google.cloud:google-cloud-bigqueryconnection to v2.2.4 (#2006) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [![WhiteSource Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com) This PR contains the following updates: | Package | Change | Age | Adoption | Passing | Confidence | |---|---|---|---|---|---| | [com.google.cloud:google-cloud-bigqueryconnection](https://togithub.com/googleapis/java-bigqueryconnection) | `2.2.3` -> `2.2.4` | [![age](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-bigqueryconnection/2.2.4/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-bigqueryconnection/2.2.4/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-bigqueryconnection/2.2.4/compatibility-slim/2.2.3)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/maven/com.google.cloud:google-cloud-bigqueryconnection/2.2.4/confidence-slim/2.2.3)](https://docs.renovatebot.com/merge-confidence/) | --- ### Release Notes
    googleapis/java-bigqueryconnection ### [`v2.2.4`](https://togithub.com/googleapis/java-bigqueryconnection/blob/HEAD/CHANGELOG.md#​224-httpsgithubcomgoogleapisjava-bigqueryconnectioncomparev223v224-2022-04-21) [Compare Source](https://togithub.com/googleapis/java-bigqueryconnection/compare/v2.2.3...v2.2.4)
    --- ### Configuration 📅 **Schedule**: At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, click this checkbox. --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#github/googleapis/java-bigquery). --- samples/install-without-bom/pom.xml | 2 +- samples/snapshot/pom.xml | 2 +- samples/snippets/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index 9d69b7909..4b298cce8 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -69,7 +69,7 @@ com.google.cloud google-cloud-bigqueryconnection - 2.2.3 + 2.2.4 test diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 2d7044933..befe43618 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -67,7 +67,7 @@ com.google.cloud google-cloud-bigqueryconnection - 2.2.3 + 2.2.4 test diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index ff98571aa..de7bbfd74 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -85,7 +85,7 @@ com.google.cloud google-cloud-bigqueryconnection - 2.2.3 + 2.2.4 test From e402471c49b696df17e34a201f4cd3ac20aa3846 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Tue, 26 Apr 2022 21:07:35 +0200 Subject: [PATCH 243/277] chore(deps): update dependency com.google.cloud:libraries-bom to v25.2.0 (#2008) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update dependency com.google.cloud:libraries-bom to v25.2.0 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot --- README.md | 4 ++-- samples/native-image-sample/pom.xml | 2 +- samples/snippets/pom.xml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index cc3ab5f44..8589bfc2b 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ See https://github.com/GoogleCloudPlatform/cloud-opensource-java/wiki/The-Google com.google.cloud libraries-bom - 25.1.0 + 25.2.0 pom import @@ -52,7 +52,7 @@ If you are using Maven without BOM, add this to your dependencies: If you are using Gradle 5.x or later, add this to your dependencies ```Groovy -implementation platform('com.google.cloud:libraries-bom:25.1.0') +implementation platform('com.google.cloud:libraries-bom:25.2.0') implementation 'com.google.cloud:google-cloud-bigquery' ``` diff --git a/samples/native-image-sample/pom.xml b/samples/native-image-sample/pom.xml index 4d22a0c99..a8955f889 100644 --- a/samples/native-image-sample/pom.xml +++ b/samples/native-image-sample/pom.xml @@ -39,7 +39,7 @@ com.google.cloud libraries-bom - 25.1.0 + 25.2.0 pom import diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index de7bbfd74..26550a1c5 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -47,7 +47,7 @@ com.google.cloud libraries-bom - 25.1.0 + 25.2.0 pom import From 4f2c3d7c8468c69aaaafa92633cc094786150fde Mon Sep 17 00:00:00 2001 From: Mridula <66699525+mpeddada1@users.noreply.github.com> Date: Wed, 27 Apr 2022 15:50:24 -0400 Subject: [PATCH 244/277] docs(sample): remove unused dependency and add setup instructions (#2010) --- samples/native-image-sample/README.md | 37 ++++++++++++++++++++++++++- samples/native-image-sample/pom.xml | 5 ---- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/samples/native-image-sample/README.md b/samples/native-image-sample/README.md index e4e4b52ee..8d7bce658 100644 --- a/samples/native-image-sample/README.md +++ b/samples/native-image-sample/README.md @@ -4,7 +4,42 @@ The BigQuery sample application demonstrates some common operations with [Google ## Setup Instructions -1. Follow the [GCP Project Authentication and Native Image Setup Instructions](../../README.md). +## Setup Instructions + +You will need to follow these prerequisite steps in order to run the samples: + +1. If you have not already, [create a Google Cloud Platform Project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#creating_a_project). + +2. Install the [Google Cloud SDK](https://cloud.google.com/sdk/) which will allow you to run the sample with your project's credentials. + + Once installed, log in with Application Default Credentials using the following command: + + ``` + gcloud auth application-default login + ``` + + **Note:** Authenticating with Application Default Credentials is convenient to use during development, but we recommend [alternate methods of authentication](https://cloud.google.com/docs/authentication/production) during production use. + +3. Install the native image compiler. + + You can follow [the installation instructions](https://www.graalvm.org/docs/getting-started/#install-graalvm) from the GraalVM website. + After following the instructions, ensure that you install the native image extension installed by running: + + ``` + gu install native-image + ``` + + Once you finish following the instructions, verify that the default version of Java is set to the GraalVM version by running `java -version` in a terminal. + + You will see something similar to the below output: + + ``` + $ java -version + + openjdk version "11.0.7" 2020-04-14 + OpenJDK Runtime Environment GraalVM CE 20.1.0 (build 11.0.7+10-jvmci-20.1-b02) + OpenJDK 64-Bit Server VM GraalVM CE 20.1.0 (build 11.0.7+10-jvmci-20.1-b02, mixed mode, sharing) + ``` 2. [Enable the BigQuery APIs](https://console.cloud.google.com/apis/api/bigquery.googleapis.com). diff --git a/samples/native-image-sample/pom.xml b/samples/native-image-sample/pom.xml index a8955f889..de1ea6adf 100644 --- a/samples/native-image-sample/pom.xml +++ b/samples/native-image-sample/pom.xml @@ -90,11 +90,6 @@ native - - com.google.cloud - native-image-support - 0.13.1 - org.junit.vintage junit-vintage-engine From 4078b9fb3c14458d2ec49e3fbeb5f81b1878a92d Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Thu, 28 Apr 2022 17:33:10 +0200 Subject: [PATCH 245/277] build(deps): update dependency org.apache.maven.plugins:maven-project-info-reports-plugin to v3.3.0 (#2011) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e466bc612..ddf7239c9 100644 --- a/pom.xml +++ b/pom.xml @@ -228,7 +228,7 @@ org.apache.maven.plugins maven-project-info-reports-plugin - 3.2.2 + 3.3.0 From 684a9da48e72516c43ace30e229eab7866017baf Mon Sep 17 00:00:00 2001 From: Stephanie Wang Date: Thu, 28 Apr 2022 11:44:40 -0700 Subject: [PATCH 246/277] chore: IT clean-up (#2012) Fixes #1924 Per discussion in b/227623980 --- .../java/com/google/cloud/bigquery/it/ITBigQueryTest.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index e4e305336..f232ee72e 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -3721,9 +3721,6 @@ public void testSnapshotTableCopyJob() throws InterruptedException { assertEquals(DDL_TABLE_SCHEMA, result.getSchema()); Table remoteTable = bigquery.getTable(DATASET, sourceTableName); assertNotNull(remoteTable); - // StandardTableDefinition tableDefinition = StandardTableDefinition.of(TABLE_SCHEMA); - // TableInfo tableInfo = TableInfo.of(sourceTableId, tableDefinition); - // Table createdTable = bigquery.create(tableInfo); // Create snapshot table using source table as the base table TableId snapshotTableId = TableId.of(DATASET, snapshotTableName); @@ -3758,10 +3755,6 @@ public void testSnapshotTableCopyJob() throws InterruptedException { .build(); Job createdRestoreJob = bigquery.create(JobInfo.of(restoreConfiguration)); CopyJobConfiguration createdRestoreConfiguration = createdRestoreJob.getConfiguration(); - // TODO: uncomment/modify below when b/227623980 is resolved - // assertEquals( - // restoreConfiguration.getSourceTables().get(0).getTable(), - // createdRestoreConfiguration.getSourceTables().get(0).getTable()); assertEquals( restoreConfiguration.getOperationType(), createdRestoreConfiguration.getOperationType()); assertEquals( From f4e2208326af5e94beb238985b3e5084ff087301 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Thu, 28 Apr 2022 23:45:47 +0200 Subject: [PATCH 247/277] build(deps): update dependency com.google.cloud:google-cloud-shared-config to v1.4.0 (#2013) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ddf7239c9..cfc9a3e9b 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,7 @@ com.google.cloud google-cloud-shared-config - 1.3.3 + 1.4.0 From 439e0ae28de7159f152452bc7386835659b8b7cd Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 29 Apr 2022 11:44:28 +0530 Subject: [PATCH 248/277] address feedback Refactored code - Replaced Tuple, Boolean> with Row Added class Row and Refactored code - Replaced Tuple, Boolean> with Row Added overloaded constructor with just message as an argument refactored and added BigQuerySQLException in next() address feedback refactored and added throwing BigQuerySQLException Added fieldvalue null checks modified close modified close modified getLong --- .../cloud/bigquery/BigQueryResultImpl.java | 207 +++++++++++------- .../cloud/bigquery/BigQuerySQLException.java | 6 + .../com/google/cloud/bigquery/Connection.java | 2 +- .../google/cloud/bigquery/ConnectionImpl.java | 29 +-- 4 files changed, 145 insertions(+), 99 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java index 527856be4..75acdf585 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java @@ -16,7 +16,6 @@ package com.google.cloud.bigquery; -import com.google.cloud.Tuple; import java.math.BigDecimal; import java.sql.Date; import java.sql.ResultSet; @@ -35,6 +34,41 @@ public class BigQueryResultImpl implements BigQueryResult { + private static final String NULL_CURSOR_MSG = + "Error occurred while reading the cursor. This could happen if getters are called after we are done reading all the records"; + + // This class represents a row of records, the columns are represented as a map + // (columnName:columnValue pair) + static class Row { + private Map value; + private boolean isLast; + + public Row(Map value) { + this.value = value; + } + + public Row(Map value, boolean isLast) { + this.value = value; + this.isLast = isLast; + } + + public Map getValue() { + return value; + } + + public boolean isLast() { + return isLast; + } + + public boolean hasField(String fieldName) { + return this.value.containsKey(fieldName); + } + + public Object get(String fieldName) { + return this.value.get(fieldName); + } + } + private final Schema schema; private final long totalRows; private final BlockingQueue buffer; @@ -80,15 +114,18 @@ public boolean next() throws SQLException { if (isEndOfStream(cursor)) { // check for end of stream cursor = null; return false; - } else if (cursor instanceof Tuple) { - Tuple, Boolean> curTup = (Tuple, Boolean>) cursor; - if (!curTup.y()) { // last Tuple + } else if (cursor instanceof Row) { + Row curTup = (Row) cursor; + if (curTup.isLast()) { // last Tuple cursor = null; return false; } + } else { // this case should never occur as the cursor will either be a Row of EoS + throw new BigQuerySQLException("Could not process the current row"); } } catch (InterruptedException e) { - throw new SQLException("Error occurred while reading buffer"); + throw new SQLException( + "Error occurred while advancing the cursor. This could happen when connection is closed while we call the next method"); } return true; @@ -104,16 +141,16 @@ public Object getObject(String fieldName) throws SQLException { throw new SQLException("fieldName can't be null"); } if (cursor == null) { - return null; + throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); - return fieldValue.getValue() == null ? null : fieldValue.getValue(); + return (fieldValue == null || fieldValue.getValue() == null) ? null : fieldValue.getValue(); } else { // Data received from Read API (Arrow) - Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; - if (!curTuple.x().containsKey(fieldName)) { + Row curRow = (Row) cursor; + if (!curRow.hasField(fieldName)) { throw new SQLException(String.format("Field %s not found", fieldName)); } - return curTuple.x().get(fieldName); + return curRow.get(fieldName); } } @@ -123,7 +160,7 @@ public Object getObject(int columnIndex) throws SQLException { return null; } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); - return fieldValue.getValue() == null ? null : fieldValue.getValue(); + return (fieldValue == null || fieldValue.getValue() == null) ? null : fieldValue.getValue(); } else { // Data received from Read API (Arrow) return getObject(schemaFieldList.get(columnIndex).getName()); } @@ -135,10 +172,10 @@ public String getString(String fieldName) throws SQLException { throw new SQLException("fieldName can't be null"); } if (cursor == null) { - return null; + throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); - if (fieldValue.getValue() == null) { + if ((fieldValue == null || fieldValue.getValue() == null)) { return null; } else if (fieldValue .getAttribute() @@ -148,11 +185,11 @@ public String getString(String fieldName) throws SQLException { return fieldValue.getStringValue(); } } else { // Data received from Read API (Arrow) - Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; - if (!curTuple.x().containsKey(fieldName)) { + Row curRow = (Row) cursor; + if (!curRow.hasField(fieldName)) { throw new SQLException(String.format("Field %s not found", fieldName)); } - Object currentVal = curTuple.x().get(fieldName); + Object currentVal = curRow.get(fieldName); if (currentVal == null) { return null; } else if (currentVal instanceof JsonStringArrayList) { // arrays @@ -174,7 +211,9 @@ public String getString(int columnIndex) throws SQLException { return null; } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); - return fieldValue.getValue() == null ? null : fieldValue.getStringValue(); + return (fieldValue == null || fieldValue.getValue() == null) + ? null + : fieldValue.getStringValue(); } else { // Data received from Read API (Arrow) return getString(schemaFieldList.get(columnIndex).getName()); } @@ -190,14 +229,16 @@ public int getInt(String fieldName) throws SQLException { // java.sql.ResultSet definition } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); - return fieldValue.getValue() == null ? 0 : fieldValue.getNumericValue().intValue(); + return (fieldValue == null || fieldValue.getValue() == null) + ? 0 + : fieldValue.getNumericValue().intValue(); } else { // Data received from Read API (Arrow) - Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; - if (!curTuple.x().containsKey(fieldName)) { + Row curRow = (Row) cursor; + if (!curRow.hasField(fieldName)) { throw new SQLException(String.format("Field %s not found", fieldName)); } - Object curVal = curTuple.x().get(fieldName); + Object curVal = curRow.get(fieldName); if (curVal == null) { return 0; } @@ -215,7 +256,9 @@ public int getInt(int columnIndex) throws SQLException { // java.sql.ResultSet definition } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); - return fieldValue.getValue() == null ? 0 : fieldValue.getNumericValue().intValue(); + return (fieldValue == null || fieldValue.getValue() == null) + ? 0 + : fieldValue.getNumericValue().intValue(); } else { // Data received from Read API (Arrow) return getInt(schemaFieldList.get(columnIndex).getName()); } @@ -227,23 +270,22 @@ public long getLong(String fieldName) throws SQLException { throw new SQLException("fieldName can't be null"); } if (cursor == null) { - return 0L; // the column value; if the value is SQL NULL, the value returned is 0 as per - // java.sql.ResultSet definition + throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); - return fieldValue.getValue() == null ? 0L : fieldValue.getNumericValue().longValue(); + return (fieldValue == null || fieldValue.getValue() == null) + ? 0L + : fieldValue.getNumericValue().longValue(); } else { // Data received from Read API (Arrow) - Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; - if (!curTuple.x().containsKey(fieldName)) { + Row curRow = (Row) cursor; + if (!curRow.hasField(fieldName)) { throw new SQLException(String.format("Field %s not found", fieldName)); } - Object curVal = curTuple.x().get(fieldName); + Object curVal = curRow.get(fieldName); if (curVal == null) { return 0L; - } else if (curVal instanceof Long) { - return ((Long) curVal).longValue(); - } else { - return ((BigDecimal) curVal).longValue(); + } else { // value will be Long or BigDecimal, but are Number + return ((Number) curVal).longValue(); } } } @@ -255,7 +297,9 @@ public long getLong(int columnIndex) throws SQLException { // java.sql.ResultSet definition } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); - return fieldValue.getValue() == null ? 0L : fieldValue.getNumericValue().longValue(); + return (fieldValue == null || fieldValue.getValue() == null) + ? 0L + : fieldValue.getNumericValue().longValue(); } else { // Data received from Read API (Arrow) return getInt(schemaFieldList.get(columnIndex).getName()); } @@ -267,17 +311,18 @@ public double getDouble(String fieldName) throws SQLException { throw new SQLException("fieldName can't be null"); } if (cursor == null) { - return 0d; // the column value; if the value is SQL NULL, the value returned is 0 as per - // java.sql.ResultSet definition + throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); - return fieldValue.getValue() == null ? 0d : fieldValue.getNumericValue().doubleValue(); + return (fieldValue == null || fieldValue.getValue() == null) + ? 0d + : fieldValue.getNumericValue().doubleValue(); } else { // Data received from Read API (Arrow) - Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; - if (!curTuple.x().containsKey(fieldName)) { + Row curRow = (Row) cursor; + if (!curRow.hasField(fieldName)) { throw new SQLException(String.format("Field %s not found", fieldName)); } - Object curVal = curTuple.x().get(fieldName); + Object curVal = curRow.get(fieldName); return curVal == null ? 0.0d : ((BigDecimal) curVal).doubleValue(); } } @@ -289,7 +334,9 @@ public double getDouble(int columnIndex) throws SQLException { // java.sql.ResultSet definition } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); - return fieldValue.getValue() == null ? 0d : fieldValue.getNumericValue().doubleValue(); + return (fieldValue == null || fieldValue.getValue() == null) + ? 0d + : fieldValue.getNumericValue().doubleValue(); } else { // Data received from Read API (Arrow) return getDouble(schemaFieldList.get(columnIndex).getName()); } @@ -301,12 +348,10 @@ public BigDecimal getBigDecimal(String fieldName) throws SQLException { throw new SQLException("fieldName can't be null"); } if (cursor == null) { - return null; // the column value (full precision); if the value is SQL NULL, the value - // returned is null in the Java programming language. as per - // java.sql.ResultSet definition + throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); - return fieldValue.getValue() == null + return (fieldValue == null || fieldValue.getValue() == null) ? null : BigDecimal.valueOf(fieldValue.getNumericValue().doubleValue()); } else { // Data received from Read API (Arrow) @@ -317,12 +362,10 @@ public BigDecimal getBigDecimal(String fieldName) throws SQLException { @Override public BigDecimal getBigDecimal(int columnIndex) throws SQLException { if (cursor == null) { - return null; // the column value (full precision); if the value is SQL NULL, the value - // returned is null in the Java programming language. as per - // java.sql.ResultSet definition + throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); - return fieldValue.getValue() == null + return (fieldValue == null || fieldValue.getValue() == null) ? null : BigDecimal.valueOf(fieldValue.getNumericValue().doubleValue()); } else { // Data received from Read API (Arrow) @@ -336,16 +379,16 @@ public boolean getBoolean(String fieldName) throws SQLException { throw new SQLException("fieldName can't be null"); } if (cursor == null) { - return false; // if the value is SQL NULL, the value returned is false + throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); return fieldValue.getValue() != null && fieldValue.getBooleanValue(); } else { // Data received from Read API (Arrow) - Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; - if (!curTuple.x().containsKey(fieldName)) { + Row curRow = (Row) cursor; + if (!curRow.hasField(fieldName)) { throw new SQLException(String.format("Field %s not found", fieldName)); } - Object curVal = curTuple.x().get(fieldName); + Object curVal = curRow.get(fieldName); return curVal != null && (Boolean) curVal; } } @@ -353,7 +396,7 @@ public boolean getBoolean(String fieldName) throws SQLException { @Override public boolean getBoolean(int columnIndex) throws SQLException { if (cursor == null) { - return false; // if the value is SQL NULL, the value returned is false + throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); return fieldValue.getValue() != null && fieldValue.getBooleanValue(); @@ -368,16 +411,18 @@ public byte[] getBytes(String fieldName) throws SQLException { throw new SQLException("fieldName can't be null"); } if (cursor == null) { - return null; // if the value is SQL NULL, the value returned is null + throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); - return fieldValue.getValue() == null ? null : fieldValue.getBytesValue(); + return (fieldValue == null || fieldValue.getValue() == null) + ? null + : fieldValue.getBytesValue(); } else { // Data received from Read API (Arrow) - Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; - if (!curTuple.x().containsKey(fieldName)) { + Row curRow = (Row) cursor; + if (!curRow.hasField(fieldName)) { throw new SQLException(String.format("Field %s not found", fieldName)); } - Object curVal = curTuple.x().get(fieldName); + Object curVal = curRow.get(fieldName); return curVal == null ? null : (byte[]) curVal; } } @@ -388,7 +433,9 @@ public byte[] getBytes(int columnIndex) throws SQLException { return null; // if the value is SQL NULL, the value returned is null } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); - return fieldValue.getValue() == null ? null : fieldValue.getBytesValue(); + return (fieldValue == null || fieldValue.getValue() == null) + ? null + : fieldValue.getBytesValue(); } else { // Data received from Read API (Arrow) return getBytes(schemaFieldList.get(columnIndex).getName()); } @@ -397,24 +444,24 @@ public byte[] getBytes(int columnIndex) throws SQLException { @Override public Timestamp getTimestamp(String fieldName) throws SQLException { if (fieldName == null) { - throw new SQLException("fieldName can't be null"); + throw new BigQuerySQLException(NULL_CURSOR_MSG); } if (cursor == null) { return null; // if the value is SQL NULL, the value returned is null } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); - return fieldValue.getValue() == null + return (fieldValue == null || fieldValue.getValue() == null) ? null : new Timestamp( fieldValue.getTimestampValue() / 1000); // getTimestampValue returns time in microseconds, and TimeStamp // expects it in millis } else { - Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; - if (!curTuple.x().containsKey(fieldName)) { + Row curRow = (Row) cursor; + if (!curRow.hasField(fieldName)) { throw new SQLException(String.format("Field %s not found", fieldName)); } - Object timeStampVal = curTuple.x().get(fieldName); + Object timeStampVal = curRow.get(fieldName); return timeStampVal == null ? null : new Timestamp((Long) timeStampVal / 1000); // Timestamp is represented as a Long @@ -424,10 +471,10 @@ public Timestamp getTimestamp(String fieldName) throws SQLException { @Override public Timestamp getTimestamp(int columnIndex) throws SQLException { if (cursor == null) { - return null; // if the value is SQL NULL, the value returned is null + throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); - return fieldValue.getValue() == null + return (fieldValue == null || fieldValue.getValue() == null) ? null : new Timestamp( fieldValue.getTimestampValue() @@ -441,7 +488,7 @@ public Timestamp getTimestamp(int columnIndex) throws SQLException { @Override public Time getTime(String fieldName) throws SQLException { if (fieldName == null) { - throw new SQLException("fieldName can't be null"); + throw new BigQuerySQLException(NULL_CURSOR_MSG); } if (cursor == null) { return null; // if the value is SQL NULL, the value returned is null @@ -449,11 +496,11 @@ public Time getTime(String fieldName) throws SQLException { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); return getTimeFromFieldVal(fieldValue); } else { // Data received from Read API (Arrow) - Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; - if (!curTuple.x().containsKey(fieldName)) { + Row curRow = (Row) cursor; + if (!curRow.hasField(fieldName)) { throw new SQLException(String.format("Field %s not found", fieldName)); } - Object timeStampObj = curTuple.x().get(fieldName); + Object timeStampObj = curRow.get(fieldName); return timeStampObj == null ? null : new Time( @@ -471,7 +518,7 @@ private int getTimeZoneOffset() { @Override public Time getTime(int columnIndex) throws SQLException { if (cursor == null) { - return null; // if the value is SQL NULL, the value returned is null + throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); return getTimeFromFieldVal(fieldValue); @@ -510,16 +557,18 @@ public Date getDate(String fieldName) throws SQLException { throw new SQLException("fieldName can't be null"); } if (cursor == null) { - return null; // if the value is SQL NULL, the value returned is null + throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(fieldName); - return fieldValue.getValue() == null ? null : Date.valueOf(fieldValue.getStringValue()); + return (fieldValue == null || fieldValue.getValue() == null) + ? null + : Date.valueOf(fieldValue.getStringValue()); } else { // Data received from Read API (Arrow) - Tuple, Boolean> curTuple = (Tuple, Boolean>) cursor; - if (!curTuple.x().containsKey(fieldName)) { + Row curRow = (Row) cursor; + if (!curRow.hasField(fieldName)) { throw new SQLException(String.format("Field %s not found", fieldName)); } - Object dateObj = curTuple.x().get(fieldName); + Object dateObj = curRow.get(fieldName); if (dateObj == null) { return null; } else { @@ -537,10 +586,12 @@ public Date getDate(String fieldName) throws SQLException { @Override public Date getDate(int columnIndex) throws SQLException { if (cursor == null) { - return null; // if the value is SQL NULL, the value returned is null + throw new BigQuerySQLException(NULL_CURSOR_MSG); } else if (cursor instanceof FieldValueList) { FieldValue fieldValue = ((FieldValueList) cursor).get(columnIndex); - return fieldValue.getValue() == null ? null : Date.valueOf(fieldValue.getStringValue()); + return (fieldValue == null || fieldValue.getValue() == null) + ? null + : Date.valueOf(fieldValue.getStringValue()); } else { // Data received from Read API (Arrow) return getDate(schemaFieldList.get(columnIndex).getName()); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuerySQLException.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuerySQLException.java index 08c6aff54..672c6ad3f 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuerySQLException.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuerySQLException.java @@ -34,6 +34,12 @@ public BigQuerySQLException() { this.errors = null; } + public BigQuerySQLException( + String msg) { // overloaded constructor with just message as an argument + super(msg); + this.errors = null; + } + public BigQuerySQLException(List errors) { this.errors = errors; } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java index 4b9479ba0..8e52da9e9 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java @@ -28,7 +28,7 @@ public interface Connection { /** Sends a query cancel request. This call will return immediately */ @BetaApi - Boolean close() throws BigQuerySQLException; + boolean close() throws BigQuerySQLException; /** * Execute a query dry run that does not return any BigQueryResult // TODO: explain more about diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index e48686d90..0eb92a6e8 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -44,7 +44,6 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.common.util.concurrent.MoreExecutors; import java.io.IOException; import java.util.AbstractList; import java.util.ArrayList; @@ -115,7 +114,7 @@ class ConnectionImpl implements Connection { */ @BetaApi @Override - public synchronized Boolean close() throws BigQuerySQLException { + public synchronized boolean close() throws BigQuerySQLException { queryTaskExecutor.shutdownNow(); try { queryTaskExecutor.awaitTermination( @@ -533,10 +532,7 @@ void populateBufferAsync( } catch (InterruptedException e) { throw new BigQueryException(0, e.getMessage(), e); } finally { - MoreExecutors.shutdownAndAwaitTermination( - queryTaskExecutor, - EXECUTOR_TIMEOUT_SEC, - TimeUnit.SECONDS); // Shutdown the thread pool + queryTaskExecutor.shutdownNow(); // Shutdown the thread pool } }; @@ -730,8 +726,7 @@ BigQueryResult highThroughPutRead( ; ReadSession readSession = bqReadClient.createReadSession(builder.build()); - BlockingQueue, Boolean>> buffer = - new LinkedBlockingDeque<>(bufferSize); + BlockingQueue buffer = new LinkedBlockingDeque<>(bufferSize); Map arrowNameToIndex = new HashMap<>(); // deserialize and populate the buffer async, so that the client isn't blocked processArrowStreamAsync( @@ -741,8 +736,7 @@ BigQueryResult highThroughPutRead( schema); logger.log(Level.INFO, "\n Using BigQuery Read API"); - return new BigQueryResultImpl, Boolean>>( - schema, totalRows, buffer, stats); + return new BigQueryResultImpl(schema, totalRows, buffer, stats); } catch (IOException e) { throw BigQueryException.translateAndThrow(e); @@ -751,7 +745,7 @@ BigQueryResult highThroughPutRead( private void processArrowStreamAsync( ReadSession readSession, - BlockingQueue, Boolean>> buffer, + BlockingQueue buffer, ArrowRowReader reader, Schema schema) { @@ -778,11 +772,8 @@ private void processArrowStreamAsync( throw BigQueryException.translateAndThrow(e); } finally { try { - buffer.put(Tuple.of(null, false)); // marking end of stream - MoreExecutors.shutdownAndAwaitTermination( - queryTaskExecutor, - EXECUTOR_TIMEOUT_SEC, - TimeUnit.SECONDS); // Shutdown the thread pool + buffer.put(new BigQueryResultImpl.Row(null, true)); // marking end of stream + queryTaskExecutor.shutdownNow(); // Shutdown the thread pool } catch (InterruptedException e) { logger.log(Level.WARNING, "\n Error occurred ", e); } @@ -822,9 +813,7 @@ private ArrowRowReader(ArrowSchema arrowSchema, Map arrowNameTo /** @param batch object returned from the ReadRowsResponse. */ private void processRows( - ArrowRecordBatch batch, - BlockingQueue, Boolean>> buffer, - Schema schema) + ArrowRecordBatch batch, BlockingQueue buffer, Schema schema) throws IOException { // deserialize the values and consume the hash of the values try { org.apache.arrow.vector.ipc.message.ArrowRecordBatch deserializedBatch = @@ -859,7 +848,7 @@ private void processRows( field.getName()); // can be accessed using the index or Vector/column name curRow.put(field.getName(), curFieldVec.getObject(rowNum)); // Added the raw value } - buffer.put(Tuple.of(curRow, true)); + buffer.put(new BigQueryResultImpl.Row(curRow)); } root.clear(); // TODO: make sure to clear the root while implementing the thread // interruption logic (Connection.close method) From eececc3d386ea8d2cf0a922bd32974cafac3683b Mon Sep 17 00:00:00 2001 From: stephwang Date: Fri, 29 Apr 2022 10:59:15 -0400 Subject: [PATCH 249/277] add clarification to dryRun query params --- .../com/google/cloud/bigquery/BigQueryDryRunResult.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResult.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResult.java index 0cc6b0891..acc045a03 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResult.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResult.java @@ -23,6 +23,10 @@ public interface BigQueryDryRunResult { /** Returns the schema of the results. Null if the schema is not supplied. */ Schema getSchema() throws BigQuerySQLException; - /** Returns query parameters for standard SQL queries */ + /** + * Returns query parameters for standard SQL queries by extracting undeclare query parameters from + * the dry run job. See more information: + * https://developers.google.com/resources/api-libraries/documentation/bigquery/v2/java/latest/com/google/api/services/bigquery/model/JobStatistics2.html#getUndeclaredQueryParameters-- + */ List getQueryParameters() throws BigQuerySQLException; } From 6c6d1738abf207008afd1f66bb16c8a7c52804d9 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Mon, 2 May 2022 12:41:30 +0530 Subject: [PATCH 250/277] Fixed BigQuerySQLException in the Integration Tests --- .../java/com/google/cloud/bigquery/BigQueryResultImpl.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java index 75acdf585..2dd9bc6dd 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java @@ -120,6 +120,9 @@ public boolean next() throws SQLException { cursor = null; return false; } + return true; + } else if (cursor instanceof FieldValueList) { // cursor is advanced, we can return true now + return true; } else { // this case should never occur as the cursor will either be a Row of EoS throw new BigQuerySQLException("Could not process the current row"); } @@ -127,8 +130,6 @@ public boolean next() throws SQLException { throw new SQLException( "Error occurred while advancing the cursor. This could happen when connection is closed while we call the next method"); } - - return true; } private boolean isEndOfStream(T cursor) { From 74b4c65a0eafb06a8a6555f1502dd6eac08d7392 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Mon, 2 May 2022 14:38:40 +0530 Subject: [PATCH 251/277] Added line comment --- .../com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java index 0e41d4b63..7e2c7d98c 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java @@ -209,6 +209,9 @@ public void testInvalidQuery() throws BigQuerySQLException { } } + /* + This tests for the order of the records as well as the value of the records using testForAllDataTypeValues + */ @Test public void testIterateAndOrder() throws SQLException { Connection connection = getConnection(); From a9c5ed1e5b98fdd2b0420195dec13300b72f43e3 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Mon, 2 May 2022 14:59:19 +0530 Subject: [PATCH 252/277] Updated getInt to include check for Long --- .../java/com/google/cloud/bigquery/BigQueryResultImpl.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java index 2dd9bc6dd..7c24ca0dd 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultImpl.java @@ -245,6 +245,10 @@ public int getInt(String fieldName) throws SQLException { } if (curVal instanceof Text) { // parse from text to int return Integer.parseInt(((Text) curVal).toString()); + } else if (curVal + instanceof + Long) { // incase getInt is called for a Long value. Loss of precision might occur + return ((Long) curVal).intValue(); } return ((BigDecimal) curVal).intValue(); } From a76843b1fd89b4434d943714860e8da62dd68e47 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Mon, 2 May 2022 14:59:57 +0530 Subject: [PATCH 253/277] Added testReadAPIIterationAndOrder IT --- .../cloud/bigquery/it/ITBigQueryTest.java | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index f232ee72e..37d8561ed 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -91,6 +91,7 @@ import com.google.cloud.bigquery.QueryJobConfiguration; import com.google.cloud.bigquery.QueryParameterValue; import com.google.cloud.bigquery.RangePartitioning; +import com.google.cloud.bigquery.ReadClientConnectionConfiguration; import com.google.cloud.bigquery.Routine; import com.google.cloud.bigquery.RoutineArgument; import com.google.cloud.bigquery.RoutineId; @@ -2668,6 +2669,50 @@ public void testBQResultSetPagination() throws SQLException { assertEquals(300000, cnt); // total 300000 rows should be read } + @Test + public void testReadAPIIterationAndOrder() + throws SQLException { // use read API to read 300K records and check the order + String query = + "SELECT date, county, state_name, confirmed_cases, deaths FROM " + + TABLE_ID_LARGE.getTable() + + " where date is not null and county is not null and state_name is not null order by confirmed_cases asc limit 300000"; + + ReadClientConnectionConfiguration clientConnectionConfiguration = + ReadClientConnectionConfiguration.newBuilder() + .setTotalToPageRowCountRatio(2L) + .setMinResultSize(100000L) + .setBufferSize(10000L) + .build(); + ConnectionSettings connectionSettings = + ConnectionSettings.newBuilder() + .setDefaultDataset(DatasetId.of(DATASET)) + .setNumBufferedRows(10000L) // page size + .setPriority( + QueryJobConfiguration.Priority + .INTERACTIVE) // so that isFastQuerySupported returns false + .setReadClientConnectionConfiguration(clientConnectionConfiguration) + .setUseReadAPI(true) + .build(); + Connection connection = bigquery.createConnection(connectionSettings); + BigQueryResult bigQueryResult = connection.executeSelect(query); + ResultSet rs = bigQueryResult.getResultSet(); + int cnt = 0; + int lasConfirmedCases = Integer.MIN_VALUE; + while (rs.next()) { // pagination starts after approx 120,000 records + assertNotNull(rs.getDate(0)); + assertNotNull(rs.getString(1)); + assertNotNull(rs.getString(2)); + assertTrue(rs.getInt(3) >= 0); + assertTrue(rs.getInt(4) >= 0); + + // check if the records are sorted + assertTrue(rs.getInt(3) >= lasConfirmedCases); + lasConfirmedCases = rs.getInt(3); + ++cnt; + } + assertEquals(300000, cnt); // total 300000 rows should be read + } + @Test public void testExecuteSelectSinglePageTableRowColInd() throws SQLException { String query = From e1acb6d5e51ed6b0ec4276a8dd90cc1e20c046e2 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 4 May 2022 18:38:40 +0530 Subject: [PATCH 254/277] Add testReadAPIConnectionMultiClose, Removed ReadClientConnectionConfiguration --- .../cloud/bigquery/it/ITBigQueryTest.java | 58 ++++++++++++++----- 1 file changed, 43 insertions(+), 15 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 37d8561ed..8072459c4 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -91,7 +91,6 @@ import com.google.cloud.bigquery.QueryJobConfiguration; import com.google.cloud.bigquery.QueryParameterValue; import com.google.cloud.bigquery.RangePartitioning; -import com.google.cloud.bigquery.ReadClientConnectionConfiguration; import com.google.cloud.bigquery.Routine; import com.google.cloud.bigquery.RoutineArgument; import com.google.cloud.bigquery.RoutineId; @@ -2509,7 +2508,7 @@ public void testBQResultSetMultiThreadedOrder() throws SQLException { ConnectionSettings connectionSettings = ConnectionSettings.newBuilder() .setDefaultDataset(DatasetId.of(DATASET)) - .setNumBufferedRows(10000L) // page size + .setNumBufferedRows(10000) // page size .build(); Connection connection = bigquery.createConnection(connectionSettings); BigQueryResult bigQueryResult = connection.executeSelect(query); @@ -2536,7 +2535,7 @@ public void testBQResultSetPaginationSlowQuery() throws SQLException { ConnectionSettings connectionSettings = ConnectionSettings.newBuilder() .setDefaultDataset(DatasetId.of(DATASET)) - .setNumBufferedRows(10000L) // page size + .setNumBufferedRows(10000) // page size .setJobTimeoutMs( 15000L) // So that ConnectionImpl.isFastQuerySupported returns false, and the slow // query route gets executed @@ -2625,7 +2624,7 @@ public void testConnectionClose() throws SQLException { ConnectionSettings connectionSettings = ConnectionSettings.newBuilder() .setDefaultDataset(DatasetId.of(DATASET)) - .setNumBufferedRows(10000L) // page size + .setNumBufferedRows(10000) // page size .build(); Connection connection = bigquery.createConnection(connectionSettings); BigQueryResult bigQueryResult = connection.executeSelect(query); @@ -2652,7 +2651,7 @@ public void testBQResultSetPagination() throws SQLException { ConnectionSettings connectionSettings = ConnectionSettings.newBuilder() .setDefaultDataset(DatasetId.of(DATASET)) - .setNumBufferedRows(10000L) // page size + .setNumBufferedRows(10000) // page size .build(); Connection connection = bigquery.createConnection(connectionSettings); BigQueryResult bigQueryResult = connection.executeSelect(query); @@ -2677,21 +2676,13 @@ public void testReadAPIIterationAndOrder() + TABLE_ID_LARGE.getTable() + " where date is not null and county is not null and state_name is not null order by confirmed_cases asc limit 300000"; - ReadClientConnectionConfiguration clientConnectionConfiguration = - ReadClientConnectionConfiguration.newBuilder() - .setTotalToPageRowCountRatio(2L) - .setMinResultSize(100000L) - .setBufferSize(10000L) - .build(); ConnectionSettings connectionSettings = ConnectionSettings.newBuilder() .setDefaultDataset(DatasetId.of(DATASET)) - .setNumBufferedRows(10000L) // page size .setPriority( QueryJobConfiguration.Priority - .INTERACTIVE) // so that isFastQuerySupported returns false - .setReadClientConnectionConfiguration(clientConnectionConfiguration) - .setUseReadAPI(true) + .INTERACTIVE) // required for this integration test so that isFastQuerySupported + // returns false .build(); Connection connection = bigquery.createConnection(connectionSettings); BigQueryResult bigQueryResult = connection.executeSelect(query); @@ -2711,6 +2702,43 @@ public void testReadAPIIterationAndOrder() ++cnt; } assertEquals(300000, cnt); // total 300000 rows should be read + connection.close(); + } + + @Test + public void testReadAPIConnectionMultiClose() + throws + SQLException { // use read API to read 300K records, then closes the connection. This test + // repeats it multiple times and assets if the connection was closed + String query = + "SELECT date, county, state_name, confirmed_cases, deaths FROM " + + TABLE_ID_LARGE.getTable() + + " where date is not null and county is not null and state_name is not null order by confirmed_cases asc limit 300000"; + + ConnectionSettings connectionSettings = + ConnectionSettings.newBuilder() + .setDefaultDataset(DatasetId.of(DATASET)) + .setPriority( + QueryJobConfiguration.Priority + .INTERACTIVE) // required for this integration test so that isFastQuerySupported + // returns false + .build(); + int closeCnt = 0, runCnt = 3; + for (int run = 0; run < runCnt; run++) { + Connection connection = bigquery.createConnection(connectionSettings); + BigQueryResult bigQueryResult = connection.executeSelect(query); + ResultSet rs = bigQueryResult.getResultSet(); + int cnt = 0; + while (rs.next()) { // pagination starts after approx 120,000 records + assertNotNull(rs.getDate(0)); + ++cnt; + } + assertEquals(300000, cnt); // total 300000 rows should be read + assertTrue(connection.close()); // check if connection closed + closeCnt++; + } + assertEquals( + closeCnt, runCnt); // check if the connection closed for the required number of times } @Test From 825fc7b088d00ab9a49f3efde4f3599296affaf7 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 4 May 2022 18:39:20 +0530 Subject: [PATCH 255/277] Refactored for removal of ReadClientConnectionConfiguration --- .../bigquery/it/ITNightlyBigQueryTest.java | 21 ++++--------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java index 7e2c7d98c..a17a53555 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java @@ -37,9 +37,7 @@ import com.google.cloud.bigquery.InsertAllRequest; import com.google.cloud.bigquery.InsertAllResponse; import com.google.cloud.bigquery.Parameter; -import com.google.cloud.bigquery.QueryJobConfiguration; import com.google.cloud.bigquery.QueryParameterValue; -import com.google.cloud.bigquery.ReadClientConnectionConfiguration; import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.StandardSQLTypeName; import com.google.cloud.bigquery.StandardTableDefinition; @@ -79,9 +77,9 @@ public class ITNightlyBigQueryTest { private static final byte[] BYTES = "TestByteValue".getBytes(StandardCharsets.UTF_8); private static final String BYTES_BASE64 = BaseEncoding.base64().encode(BYTES); // Script will populate NUM_BATCHES*REC_PER_BATCHES number of records (eg: 100*10000 = 1M) - private static final int NUM_BATCHES = 35; + private static final int NUM_BATCHES = 55; private static final int REC_PER_BATCHES = 10000; - private static final int LIMIT_RECS = 300000; // We can plan to read ~ 1M records + private static final int LIMIT_RECS = 500000; // We can plan to read ~ 500K / 1M records private static final int MULTI_LIMIT_RECS = 300000; // Used for multiquery testcase, a lower limit like 300K should be fine private static int rowCnt = 0; @@ -513,22 +511,11 @@ public static void deleteDataset(String datasetName) { } private Connection getConnection() { - ReadClientConnectionConfiguration clientConnectionConfiguration = - ReadClientConnectionConfiguration.newBuilder() - .setTotalToPageRowCountRatio(10L) - .setMinResultSize(200000L) - .setBufferSize(10000L) - .build(); + ConnectionSettings connectionSettings = ConnectionSettings.newBuilder() .setDefaultDataset(DatasetId.of(DATASET)) - .setNumBufferedRows(10000L) // page size - .setPriority( - QueryJobConfiguration.Priority - .INTERACTIVE) // so that isFastQuerySupported returns false - .setReadClientConnectionConfiguration(clientConnectionConfiguration) - .setUseReadAPI(true) - .build(); + .build(); // Read API is enabled by default return bigquery.createConnection(connectionSettings); } From cdcbe5025187462fa7a6bd959ba3a4ef73deeece Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 4 May 2022 18:39:57 +0530 Subject: [PATCH 256/277] Added more properties and default values --- .../cloud/bigquery/ConnectionSettings.java | 61 ++++++++++++++++--- 1 file changed, 53 insertions(+), 8 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java index 3a6baa413..6ee35111c 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java @@ -60,7 +60,16 @@ public abstract class ConnectionSettings { /** Returns the number of rows of data to pre-fetch */ @Nullable - public abstract Long getNumBufferedRows(); + public abstract Integer getNumBufferedRows(); + + @Nullable + public abstract Integer getTotalToPageRowCountRatio(); + + @Nullable + public abstract Integer getMinResultSize(); + + @Nullable + public abstract Integer getMaxResultPerPage(); /** Returns whether to look for the result in the query cache */ @Nullable @@ -75,9 +84,10 @@ public abstract class ConnectionSettings { @Nullable public abstract Boolean getFlattenResults(); - /** Returns the BigQuery Storage read API configuration */ - @Nullable - public abstract ReadClientConnectionConfiguration getReadClientConnectionConfiguration(); + /** + * Returns the BigQuery Storage read API configuration @Nullable public abstract + * ReadClientConnectionConfiguration getReadClientConnectionConfiguration(); + */ /** * Below properties are only supported by jobs.insert API and not yet supported by jobs.query API @@ -104,7 +114,7 @@ public abstract class ConnectionSettings { /** * Returns the table where to put query results. If not provided a new table is created. This - * value is required if {@link #allowLargeResults()} is {@code true}. + * value is required if {@link # allowLargeResults()} is {@code true}. */ @Nullable public abstract TableId getDestinationTable(); @@ -190,12 +200,21 @@ public abstract class ConnectionSettings { /** Returns a builder for a {@code ConnectionSettings} object. */ public static Builder newBuilder() { - return new AutoValue_ConnectionSettings.Builder(); + return new AutoValue_ConnectionSettings.Builder().withDefaultValues(); } @AutoValue.Builder public abstract static class Builder { + Builder withDefaultValues() { + return setUseReadAPI(true) // Read API is enabled by default + .setNumBufferedRows(10000) // 10K records will be kept in the buffer (Blocking Queue) + .setMinResultSize( + 200000) // Read API will be enabled when there will be atleast 100K records + .setTotalToPageRowCountRatio(3) // there should be atleast 3 pages of records + .setMaxResultPerPage(100000); // page size for pagination + } + /** * Sets useReadAPI flag, enabled by default. Read API will be used if the underlying conditions * are satisfied and this flag is enabled @@ -253,7 +272,30 @@ public abstract static class Builder { * * @param numBufferedRows prefetchedRowLimit or {@code null} for none */ - public abstract Builder setNumBufferedRows(Long numBufferedRows); + public abstract Builder setNumBufferedRows(Integer numBufferedRows); + + /** + * Sets a ratio of the total number of records and the records returned in the current page. + * This value is checked before calling the Read API + * + * @param totalToPageRowCountRatio totalToPageRowCountRatio + */ + public abstract Builder setTotalToPageRowCountRatio(Integer totalToPageRowCountRatio); + + /** + * Sets the minimum result size for which the Read API will be enabled + * + * @param minResultSize minResultSize + */ + public abstract Builder setMinResultSize(Integer minResultSize); + + /** + * Sets the maximum records per page to be used for pagination. This is used as an input for the + * tabledata.list + * + * @param maxResultPerPage + */ + public abstract Builder setMaxResultPerPage(Integer maxResultPerPage); /** * Sets whether to look for the result in the query cache. The query cache is a best-effort @@ -274,6 +316,8 @@ public abstract static class Builder { */ public abstract Builder setFlattenResults(Boolean flattenResults); + /* */ + /**/ /** * Sets the values necessary to determine whether table result will be read using the BigQuery * Storage client Read API. The BigQuery Storage client Read API will be used to read the query @@ -286,8 +330,9 @@ public abstract static class Builder { * * @param readClientConnectionConfiguration or {@code null} for none */ + /* public abstract Builder setReadClientConnectionConfiguration( - ReadClientConnectionConfiguration readClientConnectionConfiguration); + ReadClientConnectionConfiguration readClientConnectionConfiguration);*/ /** Sets the clustering specification for the destination table. */ public abstract Builder setClustering(Clustering clustering); From 95b52d7a80c1704676a9d12b67c2b104a6c83d4c Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 4 May 2022 18:40:31 +0530 Subject: [PATCH 257/277] Changed the type of preFetchedRowLimit from Long to Integer --- .../com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java index 2b85a966a..0626a69a7 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java @@ -533,14 +533,14 @@ public TableDataList listTableDataWithRowLimit( String projectId, String datasetId, String tableId, - Long preFetchedRowLimit, + Integer preFetchedRowLimit, String pageToken) { try { return bigquery .tabledata() .list(projectId, datasetId, tableId) .setPrettyPrint(false) - .setMaxResults(preFetchedRowLimit) + .setMaxResults(Long.valueOf(preFetchedRowLimit)) .setPageToken(pageToken) .execute(); } catch (IOException ex) { @@ -697,14 +697,14 @@ public GetQueryResultsResponse getQueryResults( @Override public GetQueryResultsResponse getQueryResultsWithRowLimit( - String projectId, String jobId, String location, Long preFetchedRowLimit) { + String projectId, String jobId, String location, Integer preFetchedRowLimit) { try { return bigquery .jobs() .getQueryResults(projectId, jobId) .setPrettyPrint(false) .setLocation(location) - .setMaxResults(preFetchedRowLimit) + .setMaxResults(Long.valueOf(preFetchedRowLimit)) .execute(); } catch (IOException ex) { throw translate(ex); From 1f01fd8f8e41ad9dcb340a6e3523b015f0c88f5a Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 4 May 2022 18:40:57 +0530 Subject: [PATCH 258/277] Updated method signature - Changed the type of preFetchedRowLimit from Long to Integer --- google-cloud-bigquery/clirr-ignored-differences.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google-cloud-bigquery/clirr-ignored-differences.xml b/google-cloud-bigquery/clirr-ignored-differences.xml index d349e0863..c8ac3b32d 100644 --- a/google-cloud-bigquery/clirr-ignored-differences.xml +++ b/google-cloud-bigquery/clirr-ignored-differences.xml @@ -20,11 +20,11 @@ 7012 com/google/cloud/bigquery/spi/v2/BigQueryRpc - com.google.api.services.bigquery.model.GetQueryResultsResponse getQueryResultsWithRowLimit(java.lang.String, java.lang.String, java.lang.String, java.lang.Long) + com.google.api.services.bigquery.model.GetQueryResultsResponse getQueryResultsWithRowLimit(java.lang.String, java.lang.String, java.lang.String, java.lang.Integer) 7012 com/google/cloud/bigquery/spi/v2/BigQueryRpc - com.google.api.services.bigquery.model.TableDataList listTableDataWithRowLimit(java.lang.String, java.lang.String, java.lang.String, java.lang.Long, java.lang.String) + com.google.api.services.bigquery.model.TableDataList listTableDataWithRowLimit(java.lang.String, java.lang.String, java.lang.String, java.lang.Integer, java.lang.String) \ No newline at end of file From fd42822464519d7642f8aeeefbd3c992c1ce9102 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 4 May 2022 18:42:02 +0530 Subject: [PATCH 259/277] Refactor - Removed the usage of ReadClientConnectionConfiguration --- .../google/cloud/bigquery/ConnectionImpl.java | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 0eb92a6e8..1b2726ee7 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -290,8 +290,8 @@ private BigQueryResult queryRpc( // Query is long-running (> 10s) and hasn't completed yet, or query completed but didn't // return the schema, fallback to jobs.insert path. Some operations don't return the schema // and can be optimized here, but this is left as future work. - long totalRows = results.getTotalRows().longValue(); - long pageRows = results.getRows().size(); + Long totalRows = results.getTotalRows() == null ? null : results.getTotalRows().longValue(); + Long pageRows = results.getRows() == null ? null : (long) (results.getRows().size()); JobId jobId = JobId.fromPb(results.getJobReference()); GetQueryResultsResponse firstPage = getQueryResultsFirstPage(jobId); return getSubsequentQueryResultsWithJob( @@ -557,7 +557,7 @@ public FieldValueList apply(TableRow rowPb) { /* Helper method that determines the optimal number of caches pages to improve read performance */ @VisibleForTesting - int getPageCacheSize(Long numBufferedRows, long numRows, Schema schema) { + int getPageCacheSize(Integer numBufferedRows, long numRows, Schema schema) { final int MIN_CACHE_SIZE = 3; // Min number of pages in the page size final int MAX_CACHE_SIZE = 20; // //Min number of pages in the page size int columnsRead = schema.getFields().size(); @@ -684,7 +684,7 @@ TableDataList tableDataListRpc(TableId destinationTable, String pageToken) { completeTableId.getProject(), completeTableId.getDataset(), completeTableId.getTable(), - connectionSettings.getNumBufferedRows(), + connectionSettings.getMaxResultPerPage(), pageToken), bigQueryOptions.getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, @@ -888,7 +888,7 @@ GetQueryResultsResponse getQueryResultsFirstPage(JobId jobId) { completeJobId.getProject(), completeJobId.getJob(), completeJobId.getLocation(), - connectionSettings.getNumBufferedRows()), + connectionSettings.getMaxResultPerPage()), bigQueryOptions.getRetrySettings(), BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, bigQueryOptions.getClock(), @@ -948,15 +948,9 @@ boolean useReadAPI(Long totalRows, Long pageRows, Schema schema, Boolean hasQuer } long resultRatio = totalRows / pageRows; - if (Boolean.TRUE.equals(connectionSettings.getUseReadAPI()) - && connectionSettings.getReadClientConnectionConfiguration() - != null) { // Adding a null check to avoid NPE - return resultRatio - > connectionSettings - .getReadClientConnectionConfiguration() - .getTotalToPageRowCountRatio() - && totalRows - > connectionSettings.getReadClientConnectionConfiguration().getMinResultSize(); + if (Boolean.TRUE.equals(connectionSettings.getUseReadAPI())) { + return resultRatio >= connectionSettings.getTotalToPageRowCountRatio() + && totalRows > connectionSettings.getMinResultSize(); } else { return false; } From 11852369ff345d2a0fb28c7ab9220d1f394fae9d Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 4 May 2022 18:42:47 +0530 Subject: [PATCH 260/277] Refactor - Removed the usage of ReadClientConnectionConfiguration --- .../cloud/bigquery/ConnectionSettingsTest.java | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionSettingsTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionSettingsTest.java index b45a11bb2..8523825bc 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionSettingsTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionSettingsTest.java @@ -33,14 +33,8 @@ public class ConnectionSettingsTest { private static final DatasetId DATASET_ID = DatasetId.of("dataset"); private static final TableId TABLE_ID = TableId.of("dataset", "table"); private static final Long REQUEST_TIMEOUT = 10l; - private static final Long NUM_BUFFERED_ROWS = 100l; + private static final Integer NUM_BUFFERED_ROWS = 100; private static final Long MAX_RESULTS = 1000l; - private static final ReadClientConnectionConfiguration READ_CLIENT_CONNECTION_CONFIGURATION = - ReadClientConnectionConfiguration.newBuilder() - .setTotalToPageRowCountRatio(3l) - .setMinResultSize(100l) - .setBufferSize(100l) - .build(); private static final List SOURCE_URIS = ImmutableList.of("uri1", "uri2"); private static final String KEY = "time_zone"; private static final String VALUE = "US/Eastern"; @@ -100,7 +94,6 @@ public class ConnectionSettingsTest { .setRequestTimeout(REQUEST_TIMEOUT) .setNumBufferedRows(NUM_BUFFERED_ROWS) .setMaxResults(MAX_RESULTS) - .setReadClientConnectionConfiguration(READ_CLIENT_CONNECTION_CONFIGURATION) .setUseQueryCache(USE_QUERY_CACHE) .setTableDefinitions(TABLE_DEFINITIONS) .setAllowLargeResults(ALLOW_LARGE_RESULTS) @@ -139,9 +132,6 @@ public void testBuilder() { assertEquals(REQUEST_TIMEOUT, CONNECTION_SETTINGS.getRequestTimeout()); assertEquals(NUM_BUFFERED_ROWS, CONNECTION_SETTINGS.getNumBufferedRows()); assertEquals(MAX_RESULTS, CONNECTION_SETTINGS.getMaxResults()); - assertEquals( - READ_CLIENT_CONNECTION_CONFIGURATION, - CONNECTION_SETTINGS.getReadClientConnectionConfiguration()); } private void compareConnectionSettings(ConnectionSettings expected, ConnectionSettings value) { @@ -151,9 +141,6 @@ private void compareConnectionSettings(ConnectionSettings expected, ConnectionSe assertEquals(expected.getRequestTimeout(), value.getRequestTimeout()); assertEquals(expected.getNumBufferedRows(), value.getNumBufferedRows()); assertEquals(expected.getMaxResults(), value.getMaxResults()); - assertEquals( - expected.getReadClientConnectionConfiguration(), - value.getReadClientConnectionConfiguration()); assertEquals(expected.getAllowLargeResults(), value.getAllowLargeResults()); assertEquals(expected.getCreateDisposition(), value.getCreateDisposition()); assertEquals(expected.getDefaultDataset(), value.getDefaultDataset()); From 35c03ae95990227d89c845b9c887b82090428541 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 4 May 2022 18:43:13 +0530 Subject: [PATCH 261/277] Changed the type of preFetchedRowLimit to Integer --- .../java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java index d19da3171..871590ca4 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java @@ -259,7 +259,7 @@ TableDataList listTableData( * @throws BigQueryException upon failure */ TableDataList listTableDataWithRowLimit( - String projectId, String datasetId, String tableId, Long rowLimit, String pageToken); + String projectId, String datasetId, String tableId, Integer rowLimit, String pageToken); /** * Returns the requested job or {@code null} if not found. @@ -315,7 +315,7 @@ GetQueryResultsResponse getQueryResults( * @throws BigQueryException upon failure */ GetQueryResultsResponse getQueryResultsWithRowLimit( - String projectId, String jobId, String location, Long preFetchedRowLimit); + String projectId, String jobId, String location, Integer preFetchedRowLimit); /** * Runs a BigQuery SQL query synchronously and returns query results if the query completes within From 6179a18a1dc0f6d9f6221425212513715ef28973 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 4 May 2022 18:43:55 +0530 Subject: [PATCH 262/277] Refactor due to Change in the type of preFetchedRowLimit to Integer --- .../google/cloud/bigquery/ConnectionImplTest.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java index 9af8ce31d..35cd10789 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java @@ -64,7 +64,7 @@ public class ConnectionImplTest { "SELECT county, state_name FROM bigquery_test_dataset.large_data_testing_table limit 2"; private static final String DRY_RUN_SQL = "SELECT county, state_name FROM bigquery_test_dataset.large_data_testing_table where country = ?"; - private static final long DEFAULT_PAGE_SIZE = 10000L; + private static final int DEFAULT_PAGE_SIZE = 10000; private ConnectionSettings connectionSettings; private static final Schema QUERY_SCHEMA = Schema.of( @@ -310,14 +310,14 @@ public void testNextPageTask() throws InterruptedException { @Test public void testGetQueryResultsFirstPage() { when(bigqueryRpcMock.getQueryResultsWithRowLimit( - any(String.class), any(String.class), any(String.class), any(Long.class))) + any(String.class), any(String.class), any(String.class), any(Integer.class))) .thenReturn(GET_QUERY_RESULTS_RESPONSE); GetQueryResultsResponse response = connection.getQueryResultsFirstPage(QUERY_JOB); assertNotNull(response); assertEquals(GET_QUERY_RESULTS_RESPONSE, response); verify(bigqueryRpcMock, times(1)) .getQueryResultsWithRowLimit( - any(String.class), any(String.class), any(String.class), any(Long.class)); + any(String.class), any(String.class), any(String.class), any(Integer.class)); } // calls executeSelect with a nonFast query and exercises createQueryJob @@ -529,9 +529,9 @@ public void testGetSubsequentQueryResultsWithJob() { public void testGetPageCacheSize() { ConnectionImpl connectionSpy = Mockito.spy(connection); // number of cached pages should be within a range - assertTrue(connectionSpy.getPageCacheSize(10000L, 100, QUERY_SCHEMA) >= 3); - assertTrue(connectionSpy.getPageCacheSize(100000000L, 100, QUERY_SCHEMA) <= 20); + assertTrue(connectionSpy.getPageCacheSize(10000, 100, QUERY_SCHEMA) >= 3); + assertTrue(connectionSpy.getPageCacheSize(100000000, 100, QUERY_SCHEMA) <= 20); verify(connectionSpy, times(2)) - .getPageCacheSize(any(Long.class), any(Long.class), any(Schema.class)); + .getPageCacheSize(any(Integer.class), any(Long.class), any(Schema.class)); } } From 260828408d03f35f19ab9861934b0a8b3af165fb Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Wed, 4 May 2022 18:44:29 +0530 Subject: [PATCH 263/277] lint --- .../test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 8072459c4..13b84c29b 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -2709,7 +2709,7 @@ public void testReadAPIIterationAndOrder() public void testReadAPIConnectionMultiClose() throws SQLException { // use read API to read 300K records, then closes the connection. This test - // repeats it multiple times and assets if the connection was closed + // repeats it multiple times and assets if the connection was closed String query = "SELECT date, county, state_name, confirmed_cases, deaths FROM " + TABLE_ID_LARGE.getTable() From e5b26c733ab9be35503250c94e6afed74f4c0a87 Mon Sep 17 00:00:00 2001 From: stephwang Date: Wed, 4 May 2022 16:51:05 -0400 Subject: [PATCH 264/277] some cleanup and corrections --- .../com/google/cloud/bigquery/BigQuery.java | 4 +- .../cloud/bigquery/BigQueryDryRunResult.java | 3 + .../cloud/bigquery/BigQueryResultStats.java | 3 + .../com/google/cloud/bigquery/Connection.java | 38 +++--- .../google/cloud/bigquery/ConnectionImpl.java | 110 +++++++++--------- .../cloud/bigquery/ConnectionSettings.java | 8 +- .../bigquery/spi/v2/HttpBigQueryRpc.java | 8 +- .../cloud/bigquery/ConnectionImplTest.java | 7 +- .../cloud/bigquery/it/ITBigQueryTest.java | 14 +++ 9 files changed, 110 insertions(+), 85 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java index 37288f9c8..798a1fd96 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java @@ -762,7 +762,8 @@ public int hashCode() { /** * Creates a new BigQuery query connection used for executing queries (not the same as BigQuery - * connection properties). + * connection properties). It uses the BigQuery Storage Read API for high throughput queries by + * default. * *

    Example of creating a query connection. * @@ -780,6 +781,7 @@ public int hashCode() { * * * @throws BigQueryException upon failure + * @param connectionSettings or null for default settings */ Connection createConnection(ConnectionSettings connectionSettings); diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResult.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResult.java index acc045a03..766ccc9d5 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResult.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResult.java @@ -16,11 +16,13 @@ package com.google.cloud.bigquery; +import com.google.api.core.BetaApi; import java.util.List; public interface BigQueryDryRunResult { /** Returns the schema of the results. Null if the schema is not supplied. */ + @BetaApi Schema getSchema() throws BigQuerySQLException; /** @@ -28,5 +30,6 @@ public interface BigQueryDryRunResult { * the dry run job. See more information: * https://developers.google.com/resources/api-libraries/documentation/bigquery/v2/java/latest/com/google/api/services/bigquery/model/JobStatistics2.html#getUndeclaredQueryParameters-- */ + @BetaApi List getQueryParameters() throws BigQuerySQLException; } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultStats.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultStats.java index aa5ffbc68..e2eb83a4a 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultStats.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultStats.java @@ -16,13 +16,16 @@ package com.google.cloud.bigquery; +import com.google.api.core.BetaApi; import com.google.cloud.bigquery.JobStatistics.SessionInfo; public interface BigQueryResultStats { /** Returns detailed statistics for DML statements. */ + @BetaApi DmlStats getDmlStats(); /** Returns SessionInfo contains information about the session if this job is part of one. */ + @BetaApi SessionInfo getSessionInfo(); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java index 8e52da9e9..109838d8b 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Connection.java @@ -31,8 +31,8 @@ public interface Connection { boolean close() throws BigQuerySQLException; /** - * Execute a query dry run that does not return any BigQueryResult // TODO: explain more about - * what this method does here + * Execute a query dry run that returns information on the schema and query parameters of the + * query results. * * @param sql typically a static SQL SELECT statement * @exception BigQuerySQLException if a database access error occurs @@ -41,30 +41,26 @@ public interface Connection { BigQueryDryRunResult dryRun(String sql) throws BigQuerySQLException; /** - * Execute a SQL statement that returns a single ResultSet + * Execute a SQL statement that returns a single ResultSet. * *

    Example of running a query. * * 

        * {
    *   @code
-   *   // ConnectionSettings connectionSettings =
-   *   //     ConnectionSettings.newBuilder()
-   *   //         .setRequestTimeout(10L)
-   *   //         .setMaxResults(100L)
-   *   //         .setUseQueryCache(true)
-   *   //         .build();
-   *   // Connection connection = bigquery.createConnection(connectionSettings);
+   *   ConnectionSettings connectionSettings =
+   *        ConnectionSettings.newBuilder()
+   *            .setRequestTimeout(10L)
+   *            .setMaxResults(100L)
+   *            .setUseQueryCache(true)
+   *            .build();
+   *   Connection connection = bigquery.createConnection(connectionSettings);
    *   String selectQuery = "SELECT corpus FROM `bigquery-public-data.samples.shakespeare` GROUP BY corpus;";
-   *   try (BigQueryResult bqResultSet = connection.executeSelect(selectQuery)) {
-   *       ResultSet rs = bqResultSet.getResultSet();
-   *       while (rs.next()) {
-   *           System.out.printf("%s,", rs.getString("corpus"));
-   *       }
-   *   } catch (BigQuerySQLException ex) {
-   *       // handle exception
+   *   BigQueryResult bqResultSet = connection.executeSelect(selectQuery)
+   *   ResultSet rs = bqResultSet.getResultSet();
+   *   while (rs.next()) {
+   *       System.out.printf("%s,", rs.getString("corpus"));
    *   }
-   * }
    *
    * * @param sql a static SQL SELECT statement @@ -80,9 +76,9 @@ public interface Connection { * @param sql SQL SELECT query * @param parameters named or positional parameters. The set of query parameters must either be * all positional or all named parameters. - * @param labels the labels associated with this query. You can use these to organize and group - * your query jobs. Label keys and values can be no longer than 63 characters, can only - * contain lowercase letters, numeric characters, underscores and dashes. International + * @param labels (optional) the labels associated with this query. You can use these to organize + * and group your query jobs. Label keys and values can be no longer than 63 characters, can + * only contain lowercase letters, numeric characters, underscores and dashes. International * characters are allowed. Label values are optional and Label is a Varargs. You should pass * all the Labels in a single Map .Label keys must start with a letter and each label in the * list must have a different key. diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 1b2726ee7..4dcbbf2aa 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -95,12 +95,13 @@ class ConnectionImpl implements Connection { this.bigQueryOptions = bigQueryOptions; this.bigQueryRpc = bigQueryRpc; this.retryConfig = retryConfig; + // Sets a reasonable buffer size (a blocking queue) if user input is suboptimal this.bufferSize = - (int) - (connectionSettings.getNumBufferedRows() == null - || connectionSettings.getNumBufferedRows() < 10000 - ? 20000 - : Math.min(connectionSettings.getNumBufferedRows() * 2, 100000)); + (connectionSettings == null + || connectionSettings.getNumBufferedRows() == null + || connectionSettings.getNumBufferedRows() < 10000 + ? 20000 + : Math.min(connectionSettings.getNumBufferedRows() * 2, 100000)); } /** @@ -325,12 +326,12 @@ BigQueryResult tableDataList(GetQueryResultsResponse firstPage, JobId jobId) { // Keeps the parsed FieldValueLists BlockingQueue, Boolean>> pageCache = new LinkedBlockingDeque<>( - getPageCacheSize(connectionSettings.getNumBufferedRows(), numRows, schema)); + getPageCacheSize(connectionSettings.getNumBufferedRows(), schema)); // Keeps the raw RPC responses BlockingQueue> rpcResponseQueue = new LinkedBlockingDeque<>( - getPageCacheSize(connectionSettings.getNumBufferedRows(), numRows, schema)); + getPageCacheSize(connectionSettings.getNumBufferedRows(), schema)); runNextPageTaskAsync(firstPage.getPageToken(), getDestinationTable(jobId), rpcResponseQueue); @@ -368,17 +369,20 @@ BigQueryResult processQueryResponseResults( BlockingQueue> buffer = new LinkedBlockingDeque<>(bufferSize); BlockingQueue, Boolean>> pageCache = new LinkedBlockingDeque<>( - getPageCacheSize(connectionSettings.getNumBufferedRows(), numRows, schema)); + getPageCacheSize(connectionSettings.getNumBufferedRows(), schema)); BlockingQueue> rpcResponseQueue = new LinkedBlockingDeque<>( - getPageCacheSize(connectionSettings.getNumBufferedRows(), numRows, schema)); + getPageCacheSize(connectionSettings.getNumBufferedRows(), schema)); JobId jobId = JobId.fromPb(results.getJobReference()); + // Thread to make rpc calls to fetch data from the server runNextPageTaskAsync(results.getPageToken(), getDestinationTable(jobId), rpcResponseQueue); + // Thread to parse data received from the server to client library objects parseRpcDataAsync(results.getRows(), schema, pageCache, rpcResponseQueue); + // Thread to populate the buffer (a blocking queue) shared with the consumer populateBufferAsync(rpcResponseQueue, pageCache, buffer); return new BigQueryResultImpl>( @@ -557,24 +561,24 @@ public FieldValueList apply(TableRow rowPb) { /* Helper method that determines the optimal number of caches pages to improve read performance */ @VisibleForTesting - int getPageCacheSize(Integer numBufferedRows, long numRows, Schema schema) { - final int MIN_CACHE_SIZE = 3; // Min number of pages in the page size - final int MAX_CACHE_SIZE = 20; // //Min number of pages in the page size - int columnsRead = schema.getFields().size(); + int getPageCacheSize(Integer numBufferedRows, Schema schema) { + final int MIN_CACHE_SIZE = 3; // Min number of pages to cache + final int MAX_CACHE_SIZE = 20; // //Min number of pages to cache + int numColumns = schema.getFields().size(); int numCachedPages; long numCachedRows = numBufferedRows == null ? 0 : numBufferedRows.longValue(); - // TODO: Further enhance this logic + // TODO: Further enhance this logic depending on customer feedback on memory consumption if (numCachedRows > 10000) { numCachedPages = 2; // the size of numBufferedRows is quite large and as per our tests we should be able to // do enough even with low - } else if (columnsRead > 15 + } else if (numColumns > 15 && numCachedRows > 5000) { // too many fields are being read, setting the page size on the lower end numCachedPages = 3; } else if (numCachedRows < 2000 - && columnsRead < 15) { // low pagesize with fewer number of columns, we can cache more pages + && numColumns < 15) { // low pagesize with fewer number of columns, we can cache more pages numCachedPages = 20; } else { // default - under 10K numCachedRows with any number of columns numCachedPages = 5; @@ -983,43 +987,45 @@ QueryRequest createQueryRequest( QueryRequest content = new QueryRequest(); String requestId = UUID.randomUUID().toString(); - if (connectionSettings.getConnectionProperties() != null) { - content.setConnectionProperties( - connectionSettings.getConnectionProperties().stream() - .map(ConnectionProperty.TO_PB_FUNCTION) - .collect(Collectors.toList())); - } - if (connectionSettings.getDefaultDataset() != null) { - content.setDefaultDataset(connectionSettings.getDefaultDataset().toPb()); - } - if (connectionSettings.getMaximumBytesBilled() != null) { - content.setMaximumBytesBilled(connectionSettings.getMaximumBytesBilled()); - } - if (connectionSettings.getMaxResults() != null) { - content.setMaxResults(connectionSettings.getMaxResults()); - } - if (queryParameters != null) { - // content.setQueryParameters(queryParameters); - if (queryParameters.get(0).getName() == null) { - // If query parameter name is unset, then assume mode is positional - content.setParameterMode("POSITIONAL"); - // pass query parameters - List queryParametersPb = - Lists.transform(queryParameters, POSITIONAL_PARAMETER_TO_PB_FUNCTION); - content.setQueryParameters(queryParametersPb); - } else { - content.setParameterMode("NAMED"); - // pass query parameters - List queryParametersPb = - Lists.transform(queryParameters, NAMED_PARAMETER_TO_PB_FUNCTION); - content.setQueryParameters(queryParametersPb); + if (connectionSettings != null) { + if (connectionSettings.getConnectionProperties() != null) { + content.setConnectionProperties( + connectionSettings.getConnectionProperties().stream() + .map(ConnectionProperty.TO_PB_FUNCTION) + .collect(Collectors.toList())); + } + if (connectionSettings.getDefaultDataset() != null) { + content.setDefaultDataset(connectionSettings.getDefaultDataset().toPb()); + } + if (connectionSettings.getMaximumBytesBilled() != null) { + content.setMaximumBytesBilled(connectionSettings.getMaximumBytesBilled()); + } + if (connectionSettings.getMaxResults() != null) { + content.setMaxResults(connectionSettings.getMaxResults()); + } + if (queryParameters != null) { + // content.setQueryParameters(queryParameters); + if (queryParameters.get(0).getName() == null) { + // If query parameter name is unset, then assume mode is positional + content.setParameterMode("POSITIONAL"); + // pass query parameters + List queryParametersPb = + Lists.transform(queryParameters, POSITIONAL_PARAMETER_TO_PB_FUNCTION); + content.setQueryParameters(queryParametersPb); + } else { + content.setParameterMode("NAMED"); + // pass query parameters + List queryParametersPb = + Lists.transform(queryParameters, NAMED_PARAMETER_TO_PB_FUNCTION); + content.setQueryParameters(queryParametersPb); + } + } + if (connectionSettings.getCreateSession() != null) { + content.setCreateSession(connectionSettings.getCreateSession()); + } + if (labels != null) { + content.setLabels(labels); } - } - if (connectionSettings.getCreateSession() != null) { - content.setCreateSession(connectionSettings.getCreateSession()); - } - if (labels != null) { - content.setLabels(labels); } content.setQuery(sql); content.setRequestId(requestId); diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java index 6ee35111c..3e45e800f 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java @@ -48,6 +48,7 @@ public abstract class ConnectionSettings { public abstract List getConnectionProperties(); /** Returns the default dataset */ + @Nullable public abstract DatasetId getDefaultDataset(); /** Returns the limits the bytes billed for this job */ @@ -268,9 +269,10 @@ Builder withDefaultValues() { public abstract Builder setMaxResults(Long maxResults); /** - * Sets the number of rows of data to pre-fetch during query execution. + * Sets the number of rows in the buffer (a blocking queue) that query results are consumed + * from. * - * @param numBufferedRows prefetchedRowLimit or {@code null} for none + * @param numBufferedRows numBufferedRows or {@code null} for none */ public abstract Builder setNumBufferedRows(Integer numBufferedRows); @@ -291,7 +293,7 @@ Builder withDefaultValues() { /** * Sets the maximum records per page to be used for pagination. This is used as an input for the - * tabledata.list + * tabledata.list and jobs.getQueryResults RPC calls * * @param maxResultPerPage */ diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java index 0626a69a7..d6b57a3da 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java @@ -533,14 +533,14 @@ public TableDataList listTableDataWithRowLimit( String projectId, String datasetId, String tableId, - Integer preFetchedRowLimit, + Integer maxResultPerPage, String pageToken) { try { return bigquery .tabledata() .list(projectId, datasetId, tableId) .setPrettyPrint(false) - .setMaxResults(Long.valueOf(preFetchedRowLimit)) + .setMaxResults(Long.valueOf(maxResultPerPage)) .setPageToken(pageToken) .execute(); } catch (IOException ex) { @@ -697,14 +697,14 @@ public GetQueryResultsResponse getQueryResults( @Override public GetQueryResultsResponse getQueryResultsWithRowLimit( - String projectId, String jobId, String location, Integer preFetchedRowLimit) { + String projectId, String jobId, String location, Integer maxResultPerPage) { try { return bigquery .jobs() .getQueryResults(projectId, jobId) .setPrettyPrint(false) .setLocation(location) - .setMaxResults(Long.valueOf(preFetchedRowLimit)) + .setMaxResults(Long.valueOf(maxResultPerPage)) .execute(); } catch (IOException ex) { throw translate(ex); diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java index 35cd10789..dd792da94 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java @@ -529,9 +529,8 @@ public void testGetSubsequentQueryResultsWithJob() { public void testGetPageCacheSize() { ConnectionImpl connectionSpy = Mockito.spy(connection); // number of cached pages should be within a range - assertTrue(connectionSpy.getPageCacheSize(10000, 100, QUERY_SCHEMA) >= 3); - assertTrue(connectionSpy.getPageCacheSize(100000000, 100, QUERY_SCHEMA) <= 20); - verify(connectionSpy, times(2)) - .getPageCacheSize(any(Integer.class), any(Long.class), any(Schema.class)); + assertTrue(connectionSpy.getPageCacheSize(10000, QUERY_SCHEMA) >= 3); + assertTrue(connectionSpy.getPageCacheSize(100000000, QUERY_SCHEMA) <= 20); + verify(connectionSpy, times(2)).getPageCacheSize(any(Integer.class), any(Schema.class)); } } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 13b84c29b..59b54137d 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -2345,6 +2345,20 @@ public void testQuery() throws InterruptedException { assertNotNull(statistics.getQueryPlan()); } + // TODO: Uncomment below test case when connectionSettings becomes optional + // @Test + // public void testExecuteSelect() throws SQLException { + // // Use default connection settings + // Connection connection = bigquery.createConnection(); + // String query = "SELECT corpus FROM `bigquery-public-data.samples.shakespeare` GROUP BY + // corpus;"; + // BigQueryResult bigQueryResult = connection.executeSelect(query); + // ResultSet rs = bigQueryResult.getResultSet(); + // while(rs.next()) { + // System.out.println(rs.getString("corpus")); + // } + // } + /* TODO(prasmish): replicate the entire test case for executeSelect */ @Test public void testQueryTimeStamp() throws InterruptedException { From 8c9d6e5f638f71fd4fb7adbd982bce31bd9d077d Mon Sep 17 00:00:00 2001 From: stephwang Date: Wed, 4 May 2022 19:02:55 -0400 Subject: [PATCH 265/277] nit --- .../com/google/cloud/bigquery/ConnectionSettings.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java index 3e45e800f..5271e1235 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java @@ -78,7 +78,7 @@ public abstract class ConnectionSettings { /** * Returns whether nested and repeated fields should be flattened. If set to {@code false} {@link - * QueryJobConfiguration.Builder#setAllowLargeResults(Boolean)} must be {@code true}. + * ConnectionSettings.Builder#setAllowLargeResults(Boolean)} must be {@code true}. * * @see Flatten */ @@ -302,8 +302,8 @@ Builder withDefaultValues() { /** * Sets whether to look for the result in the query cache. The query cache is a best-effort * cache that will be flushed whenever tables in the query are modified. Moreover, the query - * cache is only available when {@link - * QueryJobConfiguration.Builder#setDestinationTable(TableId)} is not set. + * cache is only available when {@link ConnectionSettings.Builder#setDestinationTable(TableId)} + * is not set. * * @see Query Caching */ @@ -311,7 +311,7 @@ Builder withDefaultValues() { /** * Sets whether nested and repeated fields should be flattened. If set to {@code false} {@link - * QueryJobConfiguration.Builder#setAllowLargeResults(Boolean)} must be {@code true}. By default + * ConnectionSettings.Builder#setAllowLargeResults(Boolean)} must be {@code true}. By default * results are flattened. * * @see Flatten @@ -359,7 +359,7 @@ public abstract Builder setDestinationEncryptionConfiguration( /** * Sets the table where to put query results. If not provided a new table is created. This value - * is required if {@link QueryJobConfiguration.Builder#setAllowLargeResults(Boolean)} is set to + * is required if {@link ConnectionSettings.Builder#setAllowLargeResults(Boolean)} is set to * {@code true}. */ public abstract Builder setDestinationTable(TableId destinationTable); From 58a32729949ee6a5fa39f8bce311bf7cf7f8365e Mon Sep 17 00:00:00 2001 From: stephwang Date: Wed, 4 May 2022 21:02:06 -0400 Subject: [PATCH 266/277] mark createConnection as Beta --- .../src/main/java/com/google/cloud/bigquery/BigQuery.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java index 798a1fd96..9e9217cbc 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java @@ -18,6 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument; +import com.google.api.core.BetaApi; import com.google.api.core.InternalApi; import com.google.api.gax.paging.Page; import com.google.cloud.FieldSelector; @@ -783,6 +784,7 @@ public int hashCode() { * @throws BigQueryException upon failure * @param connectionSettings or null for default settings */ + @BetaApi Connection createConnection(ConnectionSettings connectionSettings); /** From da4d7f36dec144a1c237389c8cc3c2caeab8d069 Mon Sep 17 00:00:00 2001 From: stephwang Date: Wed, 4 May 2022 21:06:17 -0400 Subject: [PATCH 267/277] disable graalvm IT until it works with Apache Arrow --- .kokoro/build.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index f5a684ca3..56f5f6336 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -69,11 +69,12 @@ integration) verify RETURN_CODE=$? ;; -graalvm) - # Run Unit and Integration Tests with Native Image - mvn -B ${INTEGRATION_TEST_ARGS} -ntp -Pnative -Penable-integration-tests test - RETURN_CODE=$? - ;; +# TODO(mpeddada): un-comment below when graalvm tests are fixed for Apache Arrow +#graalvm) +# # Run Unit and Integration Tests with Native Image +# mvn -B ${INTEGRATION_TEST_ARGS} -ntp -Pnative -Penable-integration-tests test +# RETURN_CODE=$? +# ;; samples) SAMPLES_DIR=samples # only run ITs in snapshot/ on presubmit PRs. run ITs in all 3 samples/ subdirectories otherwise. From 8f673dbbe42b0ae9fca197631dd7e7121bfbe37e Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 5 May 2022 08:51:27 +0530 Subject: [PATCH 268/277] Added instance check for IntegerArrayField --- .../cloud/bigquery/it/ITNightlyBigQueryTest.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java index a17a53555..2f59f36fa 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java @@ -407,11 +407,13 @@ private static void testForAllDataTypeValues(ResultSet rs, int cnt) throws SQLEx assertEquals("POINT(1 2)", rs.getString("GeographyField")); // Array type tests - JsonStringArrayList ary = (JsonStringArrayList) rs.getObject("IntegerArrayField"); - assertEquals(3, ary.size()); - assertEquals(1, ary.get(0).intValue()); - assertEquals(2, ary.get(1).intValue()); - assertEquals(3, ary.get(2).intValue()); + if (rs.getObject("IntegerArrayField") instanceof JsonStringArrayList) { + JsonStringArrayList ary = (JsonStringArrayList) rs.getObject("IntegerArrayField"); + assertEquals(3, ary.size()); + assertEquals(1, ary.get(0).intValue()); + assertEquals(2, ary.get(1).intValue()); + assertEquals(3, ary.get(2).intValue()); + } // BigNumeric, int and Numeric assertTrue(10000000L + cnt == rs.getDouble("BigNumericField")); From 50cc2a654700265b4e7abf6c4871e69e7f950556 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 5 May 2022 10:03:14 +0530 Subject: [PATCH 269/277] Added Connection createConnection() --- .../com/google/cloud/bigquery/BigQuery.java | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java index 9e9217cbc..fe9d0d6a5 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java @@ -33,6 +33,7 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.List; +import org.checkerframework.checker.nullness.qual.NonNull; /** * An interface for Google Cloud BigQuery. @@ -782,10 +783,29 @@ public int hashCode() { * * * @throws BigQueryException upon failure - * @param connectionSettings or null for default settings + * @param connectionSettings */ @BetaApi - Connection createConnection(ConnectionSettings connectionSettings); + Connection createConnection(@NonNull ConnectionSettings connectionSettings); + + /** + * Creates a new BigQuery query connection used for executing queries (not the same as BigQuery + * connection properties). It uses the BigQuery Storage Read API for high throughput queries by + * default. This overloaded method creates a default ConnectionSettings. + * + *

    Example of creating a query connection. + * + * 

    +   * {
+   *   @code
+   *       Connection connection = bigquery.createConnection();
+   * }
+   *
    + * + * @throws BigQueryException upon failure + */ + @BetaApi + Connection createConnection(); /** * Returns the requested dataset or {@code null} if not found. From 71dfce268160e3c1f7a7b52e0d94202e0a7e992f Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 5 May 2022 10:03:41 +0530 Subject: [PATCH 270/277] Added testIterateAndOrderDefaultConnSettings IT --- .../bigquery/it/ITNightlyBigQueryTest.java | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java index 2f59f36fa..d672967b1 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITNightlyBigQueryTest.java @@ -271,6 +271,70 @@ public void testIterateAndOrder() throws SQLException { connection.close(); } + /* + This tests for the order of the records using default connection settings as well as the value of the records using testForAllDataTypeValues + */ + @Test + public void testIterateAndOrderDefaultConnSettings() throws SQLException { + Connection connection = bigquery.createConnection(); + BigQueryResult bigQueryResult = connection.executeSelect(QUERY); + logger.log(Level.INFO, "Query used: {0}", QUERY); + ResultSet rs = bigQueryResult.getResultSet(); + int cnt = 0; + + int prevIntegerFieldVal = 0; + while (rs.next()) { + if (cnt == 0) { // first row is supposed to be null + assertNull(rs.getString("StringField")); + assertNull(rs.getString("GeographyField")); + Object intAryField = rs.getObject("IntegerArrayField"); + if (intAryField instanceof JsonStringArrayList) { + assertEquals( + new JsonStringArrayList(), + ((JsonStringArrayList) intAryField)); // null array is returned as an empty array + } + assertFalse(rs.getBoolean("BooleanField")); + assertTrue(0.0d == rs.getDouble("BigNumericField")); + assertTrue(0 == rs.getInt("IntegerField")); + assertTrue(0L == rs.getLong("NumericField")); + assertNull(rs.getBytes("BytesField")); + assertNull(rs.getTimestamp("TimestampField")); + assertNull(rs.getTime("TimeField")); + assertNull(rs.getDate("DateField")); + assertNull(rs.getString("JSONField")); + assertFalse(rs.getBoolean("BooleanField_1")); + assertNull(rs.getString("StringField_1")); + assertNull(rs.getString("hello")); // equivalent of testJsonType + assertEquals(0, rs.getInt("id")); + + } else { // remaining rows are supposed to be non null + assertNotNull(rs.getString("StringField")); + assertNotNull(rs.getString("GeographyField")); + assertNotNull(rs.getObject("IntegerArrayField")); + assertTrue(rs.getBoolean("BooleanField")); + assertTrue(0.0d < rs.getDouble("BigNumericField")); + assertTrue(0 < rs.getInt("IntegerField")); + assertTrue(0L < rs.getLong("NumericField")); + assertNotNull(rs.getBytes("BytesField")); + assertNotNull(rs.getTimestamp("TimestampField")); + assertNotNull(rs.getTime("TimeField")); + assertNotNull(rs.getDate("DateField")); + assertNotNull(rs.getString("JSONField")); + assertFalse(rs.getBoolean("BooleanField_1")); + assertNotNull(rs.getString("StringField_1")); + + // check the order of the records + assertTrue(prevIntegerFieldVal < rs.getInt("IntegerField")); + prevIntegerFieldVal = rs.getInt("IntegerField"); + + testForAllDataTypeValues(rs, cnt); // asserts the value of each row + } + ++cnt; + } + assertEquals(LIMIT_RECS, cnt); // all the records were retrieved + connection.close(); + } + @Test public void testMultipleRuns() throws SQLException { From 650e6a026a80a0380d4565339b1ac5a3816f5b0e Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 5 May 2022 10:04:44 +0530 Subject: [PATCH 271/277] Added Connection createConnection(). Marked createConnection(@NonNull ConnectionSettings connectionSettings) --- .../com/google/cloud/bigquery/BigQueryImpl.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java index bf32c3a5b..3cfbfd652 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java @@ -21,6 +21,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static java.net.HttpURLConnection.HTTP_NOT_FOUND; +import com.google.api.core.BetaApi; import com.google.api.core.InternalApi; import com.google.api.gax.paging.Page; import com.google.api.services.bigquery.model.ErrorProto; @@ -54,6 +55,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.Callable; +import org.checkerframework.checker.nullness.qual.NonNull; final class BigQueryImpl extends BaseService implements BigQuery { @@ -352,11 +354,20 @@ public JobId get() { } @Override - public Connection createConnection(ConnectionSettings connectionSettings) + @BetaApi + public Connection createConnection(@NonNull ConnectionSettings connectionSettings) throws BigQueryException { return new ConnectionImpl(connectionSettings, getOptions(), bigQueryRpc, DEFAULT_RETRY_CONFIG); } + @Override + @BetaApi + public Connection createConnection() throws BigQueryException { + ConnectionSettings defaultConnectionSettings = ConnectionSettings.newBuilder().build(); + return new ConnectionImpl( + defaultConnectionSettings, getOptions(), bigQueryRpc, DEFAULT_RETRY_CONFIG); + } + @InternalApi("visible for testing") Job create(JobInfo jobInfo, Supplier idProvider, JobOption... options) { final boolean idRandom = (jobInfo.getJobId() == null); From f4d2247e8e3bfd65330b75fe52ce4c829d628e24 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Thu, 5 May 2022 10:06:55 +0530 Subject: [PATCH 272/277] Added com.google.cloud.bigquery.Connection createConnection() --- google-cloud-bigquery/clirr-ignored-differences.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/google-cloud-bigquery/clirr-ignored-differences.xml b/google-cloud-bigquery/clirr-ignored-differences.xml index c8ac3b32d..1dccf5a66 100644 --- a/google-cloud-bigquery/clirr-ignored-differences.xml +++ b/google-cloud-bigquery/clirr-ignored-differences.xml @@ -7,6 +7,11 @@ com/google/cloud/bigquery/BigQuery com.google.cloud.bigquery.Connection createConnection(com.google.cloud.bigquery.ConnectionSettings) + + 7012 + com/google/cloud/bigquery/BigQuery + com.google.cloud.bigquery.Connection createConnection() + 7012 com/google/cloud/bigquery/spi/v2/BigQueryRpc From 6ac85f30f9453238dd364e81da592ff3c40ef174 Mon Sep 17 00:00:00 2001 From: stephwang Date: Thu, 5 May 2022 11:01:01 -0400 Subject: [PATCH 273/277] add IT and some clean up --- .../com/google/cloud/bigquery/BigQuery.java | 4 +- .../google/cloud/bigquery/ConnectionImpl.java | 74 +++++++++---------- .../cloud/bigquery/ConnectionSettings.java | 5 +- .../cloud/bigquery/it/ITBigQueryTest.java | 22 ++---- 4 files changed, 49 insertions(+), 56 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java index fe9d0d6a5..b574df32d 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java @@ -791,7 +791,9 @@ public int hashCode() { /** * Creates a new BigQuery query connection used for executing queries (not the same as BigQuery * connection properties). It uses the BigQuery Storage Read API for high throughput queries by - * default. This overloaded method creates a default ConnectionSettings. + * default. This overloaded method creates a Connection with default ConnectionSettings for query + * execution where default values are set for numBufferedRows (20000), useReadApi (true), + * useLegacySql (false). * *

    Example of creating a query connection. * diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 4dcbbf2aa..1bada1fd0 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -987,46 +987,44 @@ QueryRequest createQueryRequest( QueryRequest content = new QueryRequest(); String requestId = UUID.randomUUID().toString(); - if (connectionSettings != null) { - if (connectionSettings.getConnectionProperties() != null) { - content.setConnectionProperties( - connectionSettings.getConnectionProperties().stream() - .map(ConnectionProperty.TO_PB_FUNCTION) - .collect(Collectors.toList())); - } - if (connectionSettings.getDefaultDataset() != null) { - content.setDefaultDataset(connectionSettings.getDefaultDataset().toPb()); - } - if (connectionSettings.getMaximumBytesBilled() != null) { - content.setMaximumBytesBilled(connectionSettings.getMaximumBytesBilled()); - } - if (connectionSettings.getMaxResults() != null) { - content.setMaxResults(connectionSettings.getMaxResults()); - } - if (queryParameters != null) { - // content.setQueryParameters(queryParameters); - if (queryParameters.get(0).getName() == null) { - // If query parameter name is unset, then assume mode is positional - content.setParameterMode("POSITIONAL"); - // pass query parameters - List queryParametersPb = - Lists.transform(queryParameters, POSITIONAL_PARAMETER_TO_PB_FUNCTION); - content.setQueryParameters(queryParametersPb); - } else { - content.setParameterMode("NAMED"); - // pass query parameters - List queryParametersPb = - Lists.transform(queryParameters, NAMED_PARAMETER_TO_PB_FUNCTION); - content.setQueryParameters(queryParametersPb); - } - } - if (connectionSettings.getCreateSession() != null) { - content.setCreateSession(connectionSettings.getCreateSession()); - } - if (labels != null) { - content.setLabels(labels); + if (connectionSettings.getConnectionProperties() != null) { + content.setConnectionProperties( + connectionSettings.getConnectionProperties().stream() + .map(ConnectionProperty.TO_PB_FUNCTION) + .collect(Collectors.toList())); + } + if (connectionSettings.getDefaultDataset() != null) { + content.setDefaultDataset(connectionSettings.getDefaultDataset().toPb()); + } + if (connectionSettings.getMaximumBytesBilled() != null) { + content.setMaximumBytesBilled(connectionSettings.getMaximumBytesBilled()); + } + if (connectionSettings.getMaxResults() != null) { + content.setMaxResults(connectionSettings.getMaxResults()); + } + if (queryParameters != null) { + // content.setQueryParameters(queryParameters); + if (queryParameters.get(0).getName() == null) { + // If query parameter name is unset, then assume mode is positional + content.setParameterMode("POSITIONAL"); + // pass query parameters + List queryParametersPb = + Lists.transform(queryParameters, POSITIONAL_PARAMETER_TO_PB_FUNCTION); + content.setQueryParameters(queryParametersPb); + } else { + content.setParameterMode("NAMED"); + // pass query parameters + List queryParametersPb = + Lists.transform(queryParameters, NAMED_PARAMETER_TO_PB_FUNCTION); + content.setQueryParameters(queryParametersPb); } } + if (connectionSettings.getCreateSession() != null) { + content.setCreateSession(connectionSettings.getCreateSession()); + } + if (labels != null) { + content.setLabels(labels); + } content.setQuery(sql); content.setRequestId(requestId); // The new Connection interface only supports StandardSQL dialect diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java index 5271e1235..ac3b1b1e0 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionSettings.java @@ -210,9 +210,8 @@ public abstract static class Builder { Builder withDefaultValues() { return setUseReadAPI(true) // Read API is enabled by default .setNumBufferedRows(10000) // 10K records will be kept in the buffer (Blocking Queue) - .setMinResultSize( - 200000) // Read API will be enabled when there will be atleast 100K records - .setTotalToPageRowCountRatio(3) // there should be atleast 3 pages of records + .setMinResultSize(200000) // Read API will be enabled when there are at least 100K records + .setTotalToPageRowCountRatio(3) // there should be at least 3 pages of records .setMaxResultPerPage(100000); // page size for pagination } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 59b54137d..0ae952f57 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -2345,19 +2345,14 @@ public void testQuery() throws InterruptedException { assertNotNull(statistics.getQueryPlan()); } - // TODO: Uncomment below test case when connectionSettings becomes optional - // @Test - // public void testExecuteSelect() throws SQLException { - // // Use default connection settings - // Connection connection = bigquery.createConnection(); - // String query = "SELECT corpus FROM `bigquery-public-data.samples.shakespeare` GROUP BY - // corpus;"; - // BigQueryResult bigQueryResult = connection.executeSelect(query); - // ResultSet rs = bigQueryResult.getResultSet(); - // while(rs.next()) { - // System.out.println(rs.getString("corpus")); - // } - // } + @Test + public void testExecuteSelectDefaultConnectionSettings() throws SQLException { + // Use the default connection settings + Connection connection = bigquery.createConnection(); + String query = "SELECT corpus FROM `bigquery-public-data.samples.shakespeare` GROUP BY corpus;"; + BigQueryResult bigQueryResult = connection.executeSelect(query); + assertEquals(42, bigQueryResult.getTotalRows()); + } /* TODO(prasmish): replicate the entire test case for executeSelect */ @Test @@ -3450,7 +3445,6 @@ public void testExecuteSelectWithPositionalQueryParameters() throws BigQuerySQLE QueryParameterValue.timestamp("2014-01-01 07:00:00.000000+00:00"); Parameter stringParam = Parameter.newBuilder().setValue(stringParameter).build(); Parameter timeStampParam = Parameter.newBuilder().setValue(timestampParameter).build(); - ConnectionSettings connectionSettings = ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); Connection connection = bigquery.createConnection(connectionSettings); From adc08054c87f09dd13e55fc4e7059645bc0756ab Mon Sep 17 00:00:00 2001 From: stephwang Date: Thu, 5 May 2022 11:06:35 -0400 Subject: [PATCH 274/277] add org.checkerframework:checker-qual:jar:3.21.4:compile as a dependency to fix dependency check failure --- google-cloud-bigquery/pom.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/google-cloud-bigquery/pom.xml b/google-cloud-bigquery/pom.xml index 0e371fda6..c3abfe1cf 100644 --- a/google-cloud-bigquery/pom.xml +++ b/google-cloud-bigquery/pom.xml @@ -46,6 +46,10 @@ org.checkerframework checker-compat-qual + + org.checkerframework + checker-qual + com.google.auth google-auth-library-oauth2-http From ec764e65556b11855a67303bf28d36f57a00d07a Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Thu, 5 May 2022 15:44:02 +0000 Subject: [PATCH 275/277] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot?= =?UTF-8?q?=20post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .kokoro/build.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 56f5f6336..f5a684ca3 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -69,12 +69,11 @@ integration) verify RETURN_CODE=$? ;; -# TODO(mpeddada): un-comment below when graalvm tests are fixed for Apache Arrow -#graalvm) -# # Run Unit and Integration Tests with Native Image -# mvn -B ${INTEGRATION_TEST_ARGS} -ntp -Pnative -Penable-integration-tests test -# RETURN_CODE=$? -# ;; +graalvm) + # Run Unit and Integration Tests with Native Image + mvn -B ${INTEGRATION_TEST_ARGS} -ntp -Pnative -Penable-integration-tests test + RETURN_CODE=$? + ;; samples) SAMPLES_DIR=samples # only run ITs in snapshot/ on presubmit PRs. run ITs in all 3 samples/ subdirectories otherwise. From be30663168dcbe0f393c8d464ab73de8e0ffe246 Mon Sep 17 00:00:00 2001 From: stephwang Date: Thu, 5 May 2022 22:22:50 -0400 Subject: [PATCH 276/277] provide BigQueryResultStats in dryRun results --- .../cloud/bigquery/BigQueryDryRunResult.java | 4 ++ .../bigquery/BigQueryDryRunResultImpl.java | 11 +++++- .../cloud/bigquery/BigQueryResultStats.java | 11 ++++-- .../bigquery/BigQueryResultStatsImpl.java | 11 +++--- .../google/cloud/bigquery/ConnectionImpl.java | 38 ++++++++++++++----- .../cloud/bigquery/ConnectionImplTest.java | 16 ++++---- .../cloud/bigquery/it/ITBigQueryTest.java | 29 ++++++++++++-- 7 files changed, 90 insertions(+), 30 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResult.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResult.java index 766ccc9d5..0494aa1a9 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResult.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResult.java @@ -32,4 +32,8 @@ public interface BigQueryDryRunResult { */ @BetaApi List getQueryParameters() throws BigQuerySQLException; + + /** Returns some processing statistics */ + @BetaApi + BigQueryResultStats getStatistics() throws BigQuerySQLException; } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResultImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResultImpl.java index e8724f223..fabb2f2fc 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResultImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryDryRunResultImpl.java @@ -21,11 +21,15 @@ public class BigQueryDryRunResultImpl implements BigQueryDryRunResult { private Schema schema; private List queryParameters; + private BigQueryResultStats stats; BigQueryDryRunResultImpl( - Schema schema, List queryParameters) { // Package-Private access + Schema schema, + List queryParameters, + BigQueryResultStats stats) { // Package-Private access this.schema = schema; this.queryParameters = queryParameters; + this.stats = stats; } @Override @@ -37,4 +41,9 @@ public Schema getSchema() throws BigQuerySQLException { public List getQueryParameters() throws BigQuerySQLException { return queryParameters; } + + @Override + public BigQueryResultStats getStatistics() throws BigQuerySQLException { + return stats; + } } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultStats.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultStats.java index e2eb83a4a..a4c37a9b6 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultStats.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultStats.java @@ -17,15 +17,20 @@ package com.google.cloud.bigquery; import com.google.api.core.BetaApi; +import com.google.cloud.bigquery.JobStatistics.QueryStatistics; import com.google.cloud.bigquery.JobStatistics.SessionInfo; public interface BigQueryResultStats { - /** Returns detailed statistics for DML statements. */ + /** Returns query statistics of a query job */ @BetaApi - DmlStats getDmlStats(); + QueryStatistics getQueryStatistics(); - /** Returns SessionInfo contains information about the session if this job is part of one. */ + /** + * Returns SessionInfo contains information about the session if this job is part of one. + * JobStatistics2 model class does not allow setSessionInfo so this cannot be set as part of + * QueryStatistics when we use jobs.query API. + */ @BetaApi SessionInfo getSessionInfo(); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultStatsImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultStatsImpl.java index e0ca84576..53d67f8f3 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultStatsImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryResultStatsImpl.java @@ -16,21 +16,22 @@ package com.google.cloud.bigquery; +import com.google.cloud.bigquery.JobStatistics.QueryStatistics; import com.google.cloud.bigquery.JobStatistics.SessionInfo; public class BigQueryResultStatsImpl implements BigQueryResultStats { - private final DmlStats dmlStats; + private final QueryStatistics queryStatistics; private final SessionInfo sessionInfo; - public BigQueryResultStatsImpl(DmlStats dmlStats, SessionInfo sessionInfo) { - this.dmlStats = dmlStats; + public BigQueryResultStatsImpl(QueryStatistics queryStatistics, SessionInfo sessionInfo) { + this.queryStatistics = queryStatistics; this.sessionInfo = sessionInfo; } @Override - public DmlStats getDmlStats() { - return dmlStats; + public QueryStatistics getQueryStatistics() { + return queryStatistics; } @Override diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 1bada1fd0..c24a00888 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -28,6 +28,8 @@ import com.google.api.services.bigquery.model.TableRow; import com.google.cloud.RetryHelper; import com.google.cloud.Tuple; +import com.google.cloud.bigquery.JobStatistics.QueryStatistics; +import com.google.cloud.bigquery.JobStatistics.SessionInfo; import com.google.cloud.bigquery.spi.v2.BigQueryRpc; import com.google.cloud.bigquery.storage.v1.ArrowRecordBatch; import com.google.cloud.bigquery.storage.v1.ArrowSchema; @@ -147,7 +149,12 @@ public BigQueryDryRunResult dryRun(String sql) throws BigQuerySQLException { dryRunJob.getStatistics().getQuery().getUndeclaredQueryParameters(); List queryParameters = Lists.transform(queryParametersPb, QUERY_PARAMETER_FROM_PB_FUNCTION); - return new BigQueryDryRunResultImpl(schema, queryParameters); + QueryStatistics queryStatistics = JobStatistics.fromPb(dryRunJob); + SessionInfo sessionInfo = + queryStatistics.getSessionInfo() == null ? null : queryStatistics.getSessionInfo(); + BigQueryResultStats bigQueryResultStats = + new BigQueryResultStatsImpl(queryStatistics, sessionInfo); + return new BigQueryDryRunResultImpl(schema, queryParameters, bigQueryResultStats); } /** @@ -304,11 +311,10 @@ private BigQueryResult queryRpc( BigQueryResultStats getBigQueryResultSetStats(JobId jobId) { // Create GetQueryResultsResponse query statistics Job queryJob = getQueryJobRpc(jobId); - JobStatistics.QueryStatistics statistics = queryJob.getStatistics(); - DmlStats dmlStats = statistics.getDmlStats() == null ? null : statistics.getDmlStats(); - JobStatistics.SessionInfo sessionInfo = - statistics.getSessionInfo() == null ? null : statistics.getSessionInfo(); - return new BigQueryResultStatsImpl(dmlStats, sessionInfo); + QueryStatistics queryStatistics = queryJob.getStatistics(); + SessionInfo sessionInfo = + queryStatistics.getSessionInfo() == null ? null : queryStatistics.getSessionInfo(); + return new BigQueryResultStatsImpl(queryStatistics, sessionInfo); } /* This method processed the first page of GetQueryResultsResponse and then it uses tabledata.list */ @VisibleForTesting @@ -356,15 +362,24 @@ BigQueryResult processQueryResponseResults( Schema schema; long numRows; schema = Schema.fromPb(results.getSchema()); - numRows = results.getTotalRows().longValue(); - // Create QueryResponse query statistics + numRows = + results.getTotalRows() == null + ? 0 + : results.getTotalRows().longValue(); // in case of DML or DDL + // QueryResponse only provides cache hits, dmlStats, and sessionInfo as query processing + // statistics DmlStats dmlStats = results.getDmlStats() == null ? null : DmlStats.fromPb(results.getDmlStats()); - JobStatistics.SessionInfo sessionInfo = + Boolean cacheHit = results.getCacheHit(); + QueryStatistics queryStatistics = + QueryStatistics.newBuilder().setDmlStats(dmlStats).setCacheHit(cacheHit).build(); + // We cannot directly set sessionInfo in QueryStatistics + SessionInfo sessionInfo = results.getSessionInfo() == null ? null : JobStatistics.SessionInfo.fromPb(results.getSessionInfo()); - BigQueryResultStats bigQueryResultStats = new BigQueryResultStatsImpl(dmlStats, sessionInfo); + BigQueryResultStats bigQueryResultStats = + new BigQueryResultStatsImpl(queryStatistics, sessionInfo); BlockingQueue> buffer = new LinkedBlockingDeque<>(bufferSize); BlockingQueue, Boolean>> pageCache = @@ -1173,6 +1188,9 @@ private com.google.api.services.bigquery.model.Job createDryRunJob(String sql) { if (connectionSettings.getDefaultDataset() != null) { queryConfigurationPb.setDefaultDataset(connectionSettings.getDefaultDataset().toPb()); } + if (connectionSettings.getCreateSession() != null) { + queryConfigurationPb.setCreateSession(connectionSettings.getCreateSession()); + } configurationPb.setQuery(queryConfigurationPb); com.google.api.services.bigquery.model.Job jobPb = diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java index dd792da94..56dfe1079 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java @@ -193,14 +193,14 @@ public void testQueryDryRun() throws BigQuerySQLException { .setQuery(queryMock); com.google.api.services.bigquery.model.Job mockDryRunJob = new com.google.api.services.bigquery.model.Job().setStatistics(jobStatsMock); - - when(bigqueryRpcMock.createJobForQuery(any(com.google.api.services.bigquery.model.Job.class))) - .thenReturn(mockDryRunJob); - BigQueryDryRunResult dryRunResult = connection.dryRun(DRY_RUN_SQL); - assertEquals(1, dryRunResult.getQueryParameters().size()); - assertEquals(QUERY_SCHEMA, dryRunResult.getSchema()); - verify(bigqueryRpcMock, times(1)) - .createJobForQuery(any(com.google.api.services.bigquery.model.Job.class)); + // TODO: figure out why this test is breaking below + // when(bigqueryRpcMock.createJobForQuery(any(com.google.api.services.bigquery.model.Job.class))) + // .thenReturn(mockDryRunJob); + // BigQueryDryRunResult dryRunResult = connection.dryRun(DRY_RUN_SQL); + // assertEquals(1, dryRunResult.getQueryParameters().size()); + // assertEquals(QUERY_SCHEMA, dryRunResult.getSchema()); + // verify(bigqueryRpcMock, times(1)) + // .createJobForQuery(any(com.google.api.services.bigquery.model.Job.class)); } @Test diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 0ae952f57..348749b46 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -78,6 +78,9 @@ import com.google.cloud.bigquery.JobInfo; import com.google.cloud.bigquery.JobStatistics; import com.google.cloud.bigquery.JobStatistics.LoadStatistics; +import com.google.cloud.bigquery.JobStatistics.QueryStatistics; +import com.google.cloud.bigquery.JobStatistics.QueryStatistics.StatementType; +import com.google.cloud.bigquery.JobStatistics.SessionInfo; import com.google.cloud.bigquery.JobStatistics.TransactionInfo; import com.google.cloud.bigquery.LegacySQLTypeName; import com.google.cloud.bigquery.LoadJobConfiguration; @@ -2496,7 +2499,10 @@ public void testConnectionImplDryRun() throws SQLException { "select StringField, BigNumericField, BooleanField, BytesField, IntegerField, TimestampField, FloatField, NumericField, TimeField, DateField, DateTimeField , GeographyField, RecordField.BytesField, RecordField.BooleanField, IntegerArrayField from %s where StringField = ? order by TimestampField", TABLE_ID_FASTQUERY_BQ_RESULTSET.getTable()); ConnectionSettings connectionSettings = - ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).build(); + ConnectionSettings.newBuilder() + .setDefaultDataset(DatasetId.of(DATASET)) + .setCreateSession(true) + .build(); Connection connection = bigquery.createConnection(connectionSettings); BigQueryDryRunResult bigQueryDryRunResultSet = connection.dryRun(query); assertNotNull(bigQueryDryRunResultSet.getSchema()); @@ -2504,6 +2510,11 @@ public void testConnectionImplDryRun() throws SQLException { BQ_RESULTSET_EXPECTED_SCHEMA, bigQueryDryRunResultSet.getSchema()); // match the schema List queryParameters = bigQueryDryRunResultSet.getQueryParameters(); assertEquals(StandardSQLTypeName.STRING, queryParameters.get(0).getValue().getType()); + QueryStatistics queryStatistics = bigQueryDryRunResultSet.getStatistics().getQueryStatistics(); + assertNotNull(queryStatistics); + SessionInfo sessionInfo = bigQueryDryRunResultSet.getStatistics().getSessionInfo(); + assertNotNull(sessionInfo.getSessionId()); + assertEquals(StatementType.SELECT, queryStatistics.getStatementType()); } @Test @@ -3220,12 +3231,23 @@ public void testQuerySessionSupport() throws InterruptedException { remoteJobWithSession = remoteJobWithSession.waitFor(); assertNull(remoteJobWithSession.getStatus().getError()); Job queryJobWithSession = bigquery.getJob(remoteJobWithSession.getJobId()); - JobStatistics.QueryStatistics statisticsWithSession = queryJobWithSession.getStatistics(); + QueryStatistics statisticsWithSession = queryJobWithSession.getStatistics(); assertEquals(sessionId, statisticsWithSession.getSessionInfo().getSessionId()); } + // TODO: uncomment this testcase when executeUpdate is implemented + // @Test + // public void testExecuteSelectWithSession() throws BigQuerySQLException { + // String query = "CREATE TEMPORARY TABLE temptable AS SELECT 17 as foo"; + // ConnectionSettings connectionSettings = + // ConnectionSettings.newBuilder().setDefaultDataset(DatasetId.of(DATASET)).setCreateSession(true).build(); + // Connection connection = bigquery.createConnection(connectionSettings); + // BigQueryResult bigQueryResult = connection.execute(query); + // BigQueryResultStats stats = bigQueryResult.getBigQueryResultStats(); + // assertNotNull(stats.getSessionInfo().getSessionId()); + // } + @Test - // TODO: update this testcase when executeUpdate is implemented public void testExecuteSelectSessionSupport() throws BigQuerySQLException { String query = "SELECT 17 as foo"; ConnectionSettings connectionSettings = @@ -4081,6 +4103,7 @@ public void testQueryJobWithDryRun() throws InterruptedException, TimeoutExcepti .build(); Job remoteJob = bigquery.create(JobInfo.of(configuration)); assertNull(remoteJob.getJobId().getJob()); + remoteJob.getStatistics(); assertEquals(DONE, remoteJob.getStatus().getState()); assertNotNull(remoteJob.getConfiguration()); } From 3a5b8365558b7b978e49f459c51d7d2f1d399d95 Mon Sep 17 00:00:00 2001 From: Prashant Mishra Date: Fri, 6 May 2022 13:31:25 +0530 Subject: [PATCH 277/277] Fixed testQueryDryRun UT failure - Added jobConfigurationQuery --- .../cloud/bigquery/ConnectionImplTest.java | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java index 56dfe1079..e4fdc9731 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java @@ -191,16 +191,22 @@ public void testQueryDryRun() throws BigQuerySQLException { .setCreationTime(1234L) .setStartTime(5678L) .setQuery(queryMock); + com.google.api.services.bigquery.model.JobConfigurationQuery jobConfigurationQuery = + new com.google.api.services.bigquery.model.JobConfigurationQuery(); + com.google.api.services.bigquery.model.JobConfiguration jobConfig = + new com.google.api.services.bigquery.model.JobConfiguration() + .setQuery(jobConfigurationQuery); com.google.api.services.bigquery.model.Job mockDryRunJob = - new com.google.api.services.bigquery.model.Job().setStatistics(jobStatsMock); - // TODO: figure out why this test is breaking below - // when(bigqueryRpcMock.createJobForQuery(any(com.google.api.services.bigquery.model.Job.class))) - // .thenReturn(mockDryRunJob); - // BigQueryDryRunResult dryRunResult = connection.dryRun(DRY_RUN_SQL); - // assertEquals(1, dryRunResult.getQueryParameters().size()); - // assertEquals(QUERY_SCHEMA, dryRunResult.getSchema()); - // verify(bigqueryRpcMock, times(1)) - // .createJobForQuery(any(com.google.api.services.bigquery.model.Job.class)); + new com.google.api.services.bigquery.model.Job() + .setStatistics(jobStatsMock) + .setConfiguration(jobConfig); + when(bigqueryRpcMock.createJobForQuery(any(com.google.api.services.bigquery.model.Job.class))) + .thenReturn(mockDryRunJob); + BigQueryDryRunResult dryRunResult = connection.dryRun(DRY_RUN_SQL); + assertEquals(1, dryRunResult.getQueryParameters().size()); + assertEquals(QUERY_SCHEMA, dryRunResult.getSchema()); + verify(bigqueryRpcMock, times(1)) + .createJobForQuery(any(com.google.api.services.bigquery.model.Job.class)); } @Test