Skip to content
Permalink
Browse files
feat: adds query optimizer statistics support (#385)
* feat: adds optimizer_statistics_package option

Adds the possibility to set the optimizer statistics package when
executing queries. This option can be set in different levels as follows:
1. Through statement hints
2. Through query level configuration
3. Through an environment variable
4. Through application level configuration

If more than one package is set (in different levels) the precedence is from 1 to 4 (1 has top priority).

* test: adds integration tests

Adds integration tests for setting the query options optimizer
statistics package.

* fix: addresses PR comments

Fix missing value in the documentation of the optimizer statistics
package for the connection class.

* fix: adds tests for invalid stats packages

Adds tests for invalid statistics packages (whitespace only ones).

* fix: adds integration tests for query options

Adds an integration test to run the sql script with several expectations
for the query options. These are tests for the optimizer version and
optimizer statistics package.

* fix: formatting of ClientSideStatements.json

This file is using a mix of tabs and spaces. For now I have formatted as
the other lines, so that the diff is clear. In a further PR I will
reformat the whole file to use only spaces.

* tests: fix Connection test

Fixes connection test when using environment variables for retrieving
configurations. This only works when a first connection is created, so
we moved this specific test to its own subclass.

* fix: fix clirr checks

Provides default interface implementations and fixes clirr checks
  • Loading branch information
thiagotnunes committed Jun 5, 2021
1 parent 8a23ad0 commit e2945324783bc6d5a7a323578e8dbf00969f3163
Showing with 568 additions and 50 deletions.
  1. +19 −0 google-cloud-spanner/clirr-ignored-differences.xml
  2. +18 −0 google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java
  3. +28 −0 google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/Connection.java
  4. +14 −0 google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionImpl.java
  5. +13 −0 google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionOptions.java
  6. +4 −0 ...-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionStatementExecutor.java
  7. +16 −0 ...ud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionStatementExecutorImpl.java
  8. +2 −0 google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/StatementResult.java
  9. +23 −1 ...le-cloud-spanner/src/main/resources/com/google/cloud/spanner/connection/ClientSideStatements.json
  10. +13 −2 google-cloud-spanner/src/test/java/com/google/cloud/spanner/AbstractReadContextTest.java
  11. +27 −6 google-cloud-spanner/src/test/java/com/google/cloud/spanner/DatabaseClientImplTest.java
  12. +5 −1 google-cloud-spanner/src/test/java/com/google/cloud/spanner/SpannerImplTest.java
  13. +54 −9 google-cloud-spanner/src/test/java/com/google/cloud/spanner/SpannerOptionsTest.java
  14. +45 −0 ...e-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/AbstractConnectionImplTest.java
  15. +101 −9 google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/ConnectionImplTest.java
  16. +14 −0 ...ud-spanner/src/test/java/com/google/cloud/spanner/connection/ConnectionStatementExecutorTest.java
  17. +15 −0 ...er/src/test/java/com/google/cloud/spanner/connection/ConnectionStatementWithNoParametersTest.java
  18. +16 −0 ...er/src/test/java/com/google/cloud/spanner/connection/ConnectionStatementWithOneParameterTest.java
  19. +53 −16 google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/ConnectionTest.java
  20. +43 −0 google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/it/ITQueryOptionsTest.java
  21. +23 −5 google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITQueryOptionsTest.java
  22. +22 −1 ...anner/src/test/resources/com/google/cloud/spanner/connection/ITSqlScriptTest_TestQueryOptions.sql
@@ -618,4 +618,23 @@
<className>com/google/cloud/spanner/TransactionContext</className>
<method>com.google.api.core.ApiFuture bufferAsync(java.lang.Iterable)</method>
</difference>

<!-- Query stats optimiser statistics package -->
<!-- These are not breaking changes, since we provide default interface implementation -->
<difference>
<differenceType>7012</differenceType>
<className>com/google/cloud/spanner/SpannerOptions$SpannerEnvironment</className>
<method>java.lang.String getOptimizerStatisticsPackage()</method>
</difference>
<difference>
<differenceType>7012</differenceType>
<className>com/google/cloud/spanner/connection/Connection</className>
<method>java.lang.String getOptimizerStatisticsPackage()</method>
</difference>
<difference>
<differenceType>7012</differenceType>
<className>com/google/cloud/spanner/connection/Connection</className>
<method>void setOptimizerStatisticsPackage(java.lang.String)</method>
</difference>

</differences>
@@ -585,6 +585,15 @@ public interface SpannerEnvironment {
*/
@Nonnull
String getOptimizerVersion();

/**
* The optimizer statistics package to use. Must return an empty string to indicate that no
* value has been set.
*/
@Nonnull
default String getOptimizerStatisticsPackage() {
throw new UnsupportedOperationException("Unimplemented");
}
}

/**
@@ -594,13 +603,21 @@ public interface SpannerEnvironment {
private static class SpannerEnvironmentImpl implements SpannerEnvironment {
private static final SpannerEnvironmentImpl INSTANCE = new SpannerEnvironmentImpl();
private static final String SPANNER_OPTIMIZER_VERSION_ENV_VAR = "SPANNER_OPTIMIZER_VERSION";
private static final String SPANNER_OPTIMIZER_STATISTICS_PACKAGE_ENV_VAR =
"SPANNER_OPTIMIZER_STATISTICS_PACKAGE";

private SpannerEnvironmentImpl() {}

@Override
public String getOptimizerVersion() {
return MoreObjects.firstNonNull(System.getenv(SPANNER_OPTIMIZER_VERSION_ENV_VAR), "");
}

@Override
public String getOptimizerStatisticsPackage() {
return MoreObjects.firstNonNull(
System.getenv(SPANNER_OPTIMIZER_STATISTICS_PACKAGE_ENV_VAR), "");
}
}

/** Builder for {@link SpannerOptions} instances. */
@@ -957,6 +974,7 @@ public Builder setDefaultQueryOptions(DatabaseId database, QueryOptions defaultQ
QueryOptions getEnvironmentQueryOptions() {
return QueryOptions.newBuilder()
.setOptimizerVersion(environment.getOptimizerVersion())
.setOptimizerStatisticsPackage(environment.getOptimizerStatisticsPackage())
.build();
}

@@ -96,6 +96,11 @@
* <li><code>
* SET OPTIMIZER_VERSION='&lt;version&gt;' | 'LATEST'
* </code>: Sets the value of <code>OPTIMIZER_VERSION</code> for this connection.
* <li><code>SHOW OPTIMIZER_STATISTICS_PACKAGE</code>: Returns the current value of <code>
* OPTIMIZER_STATISTICS_PACKAGE</code> of this connection as a {@link ResultSet}
* <li><code>
* SET OPTIMIZER_STATISTICS_PACKAGE='&lt;package&gt;' | ''
* </code>: Sets the value of <code>OPTIMIZER_STATISTICS_PACKAGE</code> for this connection.
* <li><code>BEGIN [TRANSACTION]</code>: Begins a new transaction. This statement is optional when
* the connection is not in autocommit mode, as a new transaction will automatically be
* started when a query or update statement is issued. In autocommit mode, this statement will
@@ -448,6 +453,29 @@ public interface Connection extends AutoCloseable {
*/
String getOptimizerVersion();

/**
* Sets the query optimizer statistics package
*
* @param optimizerStatisticsPackage The query optimizer statistics package to use. Must be a
* string composed of letters, numbers, dashes and underscores or an empty string. The empty
* string will instruct the connection to use the optimizer statistics package that is defined
* the environment variable <code>SPANNER_OPTIMIZER_STATISTICS_PACKAGE</code>. If no value is
* specified in the environment variable, the client level query optimizer is used. If none is
* set, the default query optimizer of Cloud Spanner is used.
*/
default void setOptimizerStatisticsPackage(String optimizerStatisticsPackage) {
throw new UnsupportedOperationException("Unimplemented");
}

/**
* Gets the current query optimizer statistics package of this connection.
*
* @return The query optimizer statistics package that is currently used by this connection.
*/
default String getOptimizerStatisticsPackage() {
throw new UnsupportedOperationException("Unimplemented");
}

/**
* Sets whether this connection should request commit statistics from Cloud Spanner for read/write
* transactions and DML statements in autocommit mode.
@@ -433,6 +433,20 @@ public String getOptimizerVersion() {
return this.queryOptions.getOptimizerVersion();
}

@Override
public void setOptimizerStatisticsPackage(String optimizerStatisticsPackage) {
Preconditions.checkNotNull(optimizerStatisticsPackage);
ConnectionPreconditions.checkState(!isClosed(), CLOSED_ERROR_MSG);
this.queryOptions =
queryOptions.toBuilder().setOptimizerStatisticsPackage(optimizerStatisticsPackage).build();
}

@Override
public String getOptimizerStatisticsPackage() {
ConnectionPreconditions.checkState(!isClosed(), CLOSED_ERROR_MSG);
return this.queryOptions.getOptimizerStatisticsPackage();
}

@Override
public void setStatementTimeout(long timeout, TimeUnit unit) {
Preconditions.checkArgument(timeout > 0L, "Zero or negative timeout values are not allowed");
@@ -157,6 +157,7 @@ public String[] getValidValues() {
private static final String DEFAULT_NUM_CHANNELS = null;
private static final String DEFAULT_USER_AGENT = null;
private static final String DEFAULT_OPTIMIZER_VERSION = "";
private static final String DEFAULT_OPTIMIZER_STATISTICS_PACKAGE = "";
private static final boolean DEFAULT_RETURN_COMMIT_STATS = false;
private static final boolean DEFAULT_LENIENT = false;

@@ -190,6 +191,9 @@ public String[] getValidValues() {
private static final String USER_AGENT_PROPERTY_NAME = "userAgent";
/** Query optimizer version to use for a connection. */
private static final String OPTIMIZER_VERSION_PROPERTY_NAME = "optimizerVersion";
/** Query optimizer statistics package to use for a connection. */
private static final String OPTIMIZER_STATISTICS_PACKAGE_PROPERTY_NAME =
"optimizerStatisticsPackage";
/** Name of the 'lenientMode' connection property. */
public static final String LENIENT_PROPERTY_NAME = "lenient";

@@ -238,6 +242,8 @@ public String[] getValidValues() {
ConnectionProperty.createStringProperty(
OPTIMIZER_VERSION_PROPERTY_NAME,
"Sets the default query optimizer version to use for this connection."),
ConnectionProperty.createStringProperty(
OPTIMIZER_STATISTICS_PACKAGE_PROPERTY_NAME, ""),
ConnectionProperty.createBooleanProperty("returnCommitStats", "", false),
ConnectionProperty.createBooleanProperty(
"autoConfigEmulator",
@@ -521,6 +527,7 @@ private ConnectionOptions(Builder builder) {
this.userAgent = parseUserAgent(this.uri);
QueryOptions.Builder queryOptionsBuilder = QueryOptions.newBuilder();
queryOptionsBuilder.setOptimizerVersion(parseOptimizerVersion(this.uri));
queryOptionsBuilder.setOptimizerStatisticsPackage(parseOptimizerStatisticsPackage(this.uri));
this.queryOptions = queryOptionsBuilder.build();
this.returnCommitStats = parseReturnCommitStats(this.uri);
this.autoConfigEmulator = parseAutoConfigEmulator(this.uri);
@@ -695,6 +702,12 @@ static String parseOptimizerVersion(String uri) {
return value != null ? value : DEFAULT_OPTIMIZER_VERSION;
}

@VisibleForTesting
static String parseOptimizerStatisticsPackage(String uri) {
String value = parseUriProperty(uri, OPTIMIZER_STATISTICS_PACKAGE_PROPERTY_NAME);
return value != null ? value : DEFAULT_OPTIMIZER_STATISTICS_PACKAGE;
}

@VisibleForTesting
static boolean parseReturnCommitStats(String uri) {
String value = parseUriProperty(uri, "returnCommitStats");
@@ -66,6 +66,10 @@ interface ConnectionStatementExecutor {

StatementResult statementShowOptimizerVersion();

StatementResult statementSetOptimizerStatisticsPackage(String optimizerStatisticsPackage);

StatementResult statementShowOptimizerStatisticsPackage();

StatementResult statementSetReturnCommitStats(Boolean returnCommitStats);

StatementResult statementShowReturnCommitStats();
@@ -23,6 +23,7 @@
import static com.google.cloud.spanner.connection.StatementResult.ClientSideStatementType.RUN_BATCH;
import static com.google.cloud.spanner.connection.StatementResult.ClientSideStatementType.SET_AUTOCOMMIT;
import static com.google.cloud.spanner.connection.StatementResult.ClientSideStatementType.SET_AUTOCOMMIT_DML_MODE;
import static com.google.cloud.spanner.connection.StatementResult.ClientSideStatementType.SET_OPTIMIZER_STATISTICS_PACKAGE;
import static com.google.cloud.spanner.connection.StatementResult.ClientSideStatementType.SET_OPTIMIZER_VERSION;
import static com.google.cloud.spanner.connection.StatementResult.ClientSideStatementType.SET_READONLY;
import static com.google.cloud.spanner.connection.StatementResult.ClientSideStatementType.SET_READ_ONLY_STALENESS;
@@ -34,6 +35,7 @@
import static com.google.cloud.spanner.connection.StatementResult.ClientSideStatementType.SHOW_AUTOCOMMIT_DML_MODE;
import static com.google.cloud.spanner.connection.StatementResult.ClientSideStatementType.SHOW_COMMIT_RESPONSE;
import static com.google.cloud.spanner.connection.StatementResult.ClientSideStatementType.SHOW_COMMIT_TIMESTAMP;
import static com.google.cloud.spanner.connection.StatementResult.ClientSideStatementType.SHOW_OPTIMIZER_STATISTICS_PACKAGE;
import static com.google.cloud.spanner.connection.StatementResult.ClientSideStatementType.SHOW_OPTIMIZER_VERSION;
import static com.google.cloud.spanner.connection.StatementResult.ClientSideStatementType.SHOW_READONLY;
import static com.google.cloud.spanner.connection.StatementResult.ClientSideStatementType.SHOW_READ_ONLY_STALENESS;
@@ -230,6 +232,20 @@ public StatementResult statementShowOptimizerVersion() {
"OPTIMIZER_VERSION", getConnection().getOptimizerVersion(), SHOW_OPTIMIZER_VERSION);
}

@Override
public StatementResult statementSetOptimizerStatisticsPackage(String optimizerStatisticsPackage) {
getConnection().setOptimizerStatisticsPackage(optimizerStatisticsPackage);
return noResult(SET_OPTIMIZER_STATISTICS_PACKAGE);
}

@Override
public StatementResult statementShowOptimizerStatisticsPackage() {
return resultSet(
"OPTIMIZER_STATISTICS_PACKAGE",
getConnection().getOptimizerStatisticsPackage(),
SHOW_OPTIMIZER_STATISTICS_PACKAGE);
}

@Override
public StatementResult statementSetReturnCommitStats(Boolean returnCommitStats) {
getConnection().setReturnCommitStats(returnCommitStats);
@@ -65,6 +65,8 @@ enum ClientSideStatementType {
SET_READ_ONLY_STALENESS,
SHOW_OPTIMIZER_VERSION,
SET_OPTIMIZER_VERSION,
SHOW_OPTIMIZER_STATISTICS_PACKAGE,
SET_OPTIMIZER_STATISTICS_PACKAGE,
SHOW_RETURN_COMMIT_STATS,
SET_RETURN_COMMIT_STATS,
BEGIN,
@@ -76,6 +76,14 @@
"method": "statementShowOptimizerVersion",
"exampleStatements": ["show variable optimizer_version"]
},
{
"name": "SHOW VARIABLE OPTIMIZER_STATISTICS_PACKAGE",
"executorName": "ClientSideStatementNoParamExecutor",
"resultType": "RESULT_SET",
"regex": "(?is)\\A\\s*show\\s+variable\\s+optimizer_statistics_package\\s*\\z",
"method": "statementShowOptimizerStatisticsPackage",
"exampleStatements": ["show variable optimizer_statistics_package"]
},
{
"name": "SHOW VARIABLE RETURN_COMMIT_STATS",
"executorName": "ClientSideStatementNoParamExecutor",
@@ -281,6 +289,20 @@
"converterName": "ClientSideStatementValueConverters$StringValueConverter"
}
},
{
"name": "SET OPTIMIZER_STATISTICS_PACKAGE = '<package>'|''",
"executorName": "ClientSideStatementSetExecutor",
"resultType": "NO_RESULT",
"regex": "(?is)\\A\\s*set\\s+optimizer_statistics_package\\s*(?:=)\\s*(.*)\\z",
"method": "statementSetOptimizerStatisticsPackage",
"exampleStatements": ["set optimizer_statistics_package='auto_20191128_14_47_22UTC'", "set optimizer_statistics_package=''"],
"setStatement": {
"propertyName": "OPTIMIZER_STATISTICS_PACKAGE",
"separator": "=",
"allowedValues": "'((\\S+)|())'",
"converterName": "ClientSideStatementValueConverters$StringValueConverter"
}
},
{
"name": "SET RETURN_COMMIT_STATS = TRUE|FALSE",
"executorName": "ClientSideStatementSetExecutor",
@@ -296,4 +318,4 @@
}
}
]
}
}
@@ -52,7 +52,12 @@ public static Collection<Object[]> parameters() {
List<Object[]> params = new ArrayList<>();
params.add(new Object[] {QueryOptions.getDefaultInstance()});
params.add(
new Object[] {QueryOptions.newBuilder().setOptimizerVersion("some-version").build()});
new Object[] {
QueryOptions.newBuilder()
.setOptimizerVersion("some-version")
.setOptimizerStatisticsPackage("some-package")
.build()
});
return params;
}

@@ -134,14 +139,20 @@ public void executeSqlRequestBuilderWithQueryOptions() {
context
.getExecuteSqlRequestBuilder(
Statement.newBuilder("SELECT FOO FROM BAR")
.withQueryOptions(QueryOptions.newBuilder().setOptimizerVersion("2.0").build())
.withQueryOptions(
QueryOptions.newBuilder()
.setOptimizerVersion("2.0")
.setOptimizerStatisticsPackage("custom-package")
.build())
.build(),
QueryMode.NORMAL,
Options.fromQueryOptions(),
true)
.build();
assertThat(request.getSql()).isEqualTo("SELECT FOO FROM BAR");
assertThat(request.getQueryOptions().getOptimizerVersion()).isEqualTo("2.0");
assertThat(request.getQueryOptions().getOptimizerStatisticsPackage())
.isEqualTo("custom-package");
}

@Test

0 comments on commit e294532

Please sign in to comment.