From c7f85f3e6c575b89a0ff37810c08e7eef6774ee0 Mon Sep 17 00:00:00 2001 From: Praful Makani Date: Fri, 11 Oct 2019 17:10:41 +0530 Subject: [PATCH 1/3] added RangePartitioning feature to LoadJobConfiguration --- .../cloud/bigquery/LoadJobConfiguration.java | 30 ++- .../cloud/bigquery/RangePartitioning.java | 241 ++++++++++++++++++ .../bigquery/LoadJobConfigurationTest.java | 9 + 3 files changed, 279 insertions(+), 1 deletion(-) create mode 100644 google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/RangePartitioning.java diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java index 01593accbace..ff537d98a3a1 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java @@ -53,6 +53,7 @@ public final class LoadJobConfiguration extends JobConfiguration implements Load private final Boolean useAvroLogicalTypes; private final Map labels; private final Long jobTimeoutMs; + private final RangePartitioning rangePartitioning; public static final class Builder extends JobConfiguration.Builder implements LoadConfiguration.Builder { @@ -75,6 +76,7 @@ public static final class Builder extends JobConfiguration.Builder labels; private Long jobTimeoutMs; + private RangePartitioning rangePartitioning; private Builder() { super(Type.LOAD); @@ -100,6 +102,7 @@ private Builder(LoadJobConfiguration loadConfiguration) { this.useAvroLogicalTypes = loadConfiguration.useAvroLogicalTypes; this.labels = loadConfiguration.labels; this.jobTimeoutMs = loadConfiguration.jobTimeoutMs; + this.rangePartitioning = loadConfiguration.rangePartitioning; } private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) { @@ -179,6 +182,10 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur if (configurationPb.getJobTimeoutMs() != null) { this.jobTimeoutMs = configurationPb.getJobTimeoutMs(); } + if (loadConfigurationPb.getRangePartitioning() != null) { + this.rangePartitioning = + RangePartitioning.fromPb(loadConfigurationPb.getRangePartitioning()); + } } @Override @@ -301,6 +308,17 @@ public Builder setJobTimeoutMs(Long jobTimeoutMs) { return this; } + /** + * Range partitioning specification for this table. Only one of timePartitioning and + * rangePartitioning should be specified. + * + * @param rangePartitioning rangePartitioning or {@code null} for none + */ + public Builder setRangePartitioning(RangePartitioning rangePartitioning) { + this.rangePartitioning = rangePartitioning; + return this; + } + @Override public LoadJobConfiguration build() { return new LoadJobConfiguration(this); @@ -326,6 +344,7 @@ private LoadJobConfiguration(Builder builder) { this.useAvroLogicalTypes = builder.useAvroLogicalTypes; this.labels = builder.labels; this.jobTimeoutMs = builder.jobTimeoutMs; + this.rangePartitioning = builder.rangePartitioning; } @Override @@ -428,6 +447,11 @@ public Long getJobTimeoutMs() { return jobTimeoutMs; } + /** Returns the range partitioning specification for this table */ + public RangePartitioning getRangePartitioning() { + return rangePartitioning; + } + @Override public Builder toBuilder() { return new Builder(this); @@ -452,7 +476,8 @@ ToStringHelper toStringHelper() { .add("clustering", clustering) .add("useAvroLogicalTypes", useAvroLogicalTypes) .add("labels", labels) - .add("jobTimeoutMs", jobTimeoutMs); + .add("jobTimeoutMs", jobTimeoutMs) + .add("rangePartitioning", rangePartitioning); } @Override @@ -542,6 +567,9 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() { if (jobTimeoutMs != null) { jobConfiguration.setJobTimeoutMs(jobTimeoutMs); } + if (rangePartitioning != null) { + loadConfigurationPb.setRangePartitioning(rangePartitioning.toPb()); + } jobConfiguration.setLoad(loadConfigurationPb); return jobConfiguration; } diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/RangePartitioning.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/RangePartitioning.java new file mode 100644 index 000000000000..54529d0bad54 --- /dev/null +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/RangePartitioning.java @@ -0,0 +1,241 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigquery; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Strings.isNullOrEmpty; + +import com.google.common.base.MoreObjects; +import com.google.common.base.Objects; +import java.io.Serializable; + +public final class RangePartitioning implements Serializable { + + private static final long serialVersionUID = 2011646901758026886L; + private final String field; + private final Range range; + + public static final class Range implements Serializable { + + private static final long serialVersionUID = -7603436109151103007L; + private final Long end; + private final Long interval; + private final Long start; + + private Range(Builder builder) { + this.end = builder.end; + this.interval = builder.interval; + this.start = builder.start; + } + + private Range(com.google.api.services.bigquery.model.RangePartitioning.Range range) { + this.end = range.getEnd(); + this.interval = range.getInterval(); + this.start = range.getStart(); + } + + /** A builder for {@code Range} objects. */ + public static final class Builder { + + private Long end; + private Long interval; + private Long start; + + private Builder() {} + + /** [Required] The end of range partitioning, exclusive. The value may be {@code null}. */ + public Builder setEnd(Long end) { + this.end = end; + return this; + } + + /** [Required] The width of each interval. The value may be {@code null}. */ + public Builder setInterval(Long interval) { + this.interval = interval; + return this; + } + + /** [Required] The start of range partitioning, inclusive. The value may be {@code null}. */ + public Builder setStart(Long start) { + this.start = start; + return this; + } + + /** Creates a {@code Range} object. */ + public Range build() { + return new Range(this); + } + } + + /** Returns the end of range partitioning. */ + public Long getEnd() { + return end; + } + + /** Returns the width of each interval. */ + public Long getInterval() { + return interval; + } + + /** Returns the start of range partitioning. */ + public Long getStart() { + return start; + } + + /** Returns a builder for a Range object. */ + public static Builder newBuilder() { + return new Builder(); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("end", end) + .add("interval", interval) + .add("start", start) + .toString(); + } + + @Override + public int hashCode() { + return Objects.hashCode(end, interval, start); + } + + @Override + public boolean equals(Object obj) { + return obj == this + || obj != null + && obj.getClass().equals(Range.class) + && java.util.Objects.equals(toPb(), ((Range) obj).toPb()); + } + + com.google.api.services.bigquery.model.RangePartitioning.Range toPb() { + com.google.api.services.bigquery.model.RangePartitioning.Range range = + new com.google.api.services.bigquery.model.RangePartitioning.Range(); + range.setEnd(end); + range.setInterval(interval); + range.setStart(start); + return range; + } + + static Range fromPb(com.google.api.services.bigquery.model.RangePartitioning.Range range) { + return new Range(range); + } + } + + /** A builder for {@code RangePartitioning} objects. */ + public static final class Builder { + + private String field; + private Range range; + + private Builder() {} + + private Builder(RangePartitioning rangePartitioning) { + this.field = rangePartitioning.field; + this.range = rangePartitioning.range; + } + + /** + * [Required] The table is partitioned by this field. The field must be a top- level + * NULLABLE/REQUIRED field. The only supported type is INTEGER/INT64. + * + * @param field field or {@code null} for none + */ + public Builder setField(String field) { + checkArgument(!isNullOrEmpty(field), "Provided field is null or empty"); + this.field = field; + return this; + } + + /** + * [Required] Defines the ranges for range partitioning. + * + * @param range range or {@code null} for none + */ + public Builder setRange(Range range) { + this.range = range; + return this; + } + + /** Creates a {@code RangePartitioning} object. */ + public RangePartitioning build() { + return new RangePartitioning(this); + } + } + + private RangePartitioning(Builder builder) { + this.field = builder.field; + this.range = builder.range; + } + + private RangePartitioning( + com.google.api.services.bigquery.model.RangePartitioning rangePartitioning) { + this.field = rangePartitioning.getField(); + this.range = Range.fromPb(rangePartitioning.getRange()); + } + + /** Returns the field of range partitioning. */ + public String getField() { + return field; + } + + /** Returns the range of range partitioning. */ + public Range getRange() { + return range; + } + + /** Returns a builder for a RangePartitioning object */ + public static Builder newBuilder() { + return new Builder(); + } + + /** Returns a builder for the {@code RangePartitioning} object. */ + public Builder toBuilder() { + return new Builder(this); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this).add("field", field).add("range", range).toString(); + } + + @Override + public int hashCode() { + return Objects.hashCode(field, range); + } + + @Override + public boolean equals(Object obj) { + return obj == this + || obj != null + && obj.getClass().equals(RangePartitioning.class) + && java.util.Objects.equals(toPb(), ((RangePartitioning) obj).toPb()); + } + + com.google.api.services.bigquery.model.RangePartitioning toPb() { + com.google.api.services.bigquery.model.RangePartitioning rangePartitioning = + new com.google.api.services.bigquery.model.RangePartitioning(); + rangePartitioning.setField(field); + rangePartitioning.setRange(range.toPb()); + return rangePartitioning; + } + + static RangePartitioning fromPb( + com.google.api.services.bigquery.model.RangePartitioning rangePartitioning) { + return new RangePartitioning(rangePartitioning); + } +} diff --git a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java index 3ef06a4027d6..b64e3c451419 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java +++ b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java @@ -63,6 +63,10 @@ public class LoadJobConfigurationTest { private static final Map LABELS = ImmutableMap.of("test-job-name", "test-load-job"); private static final Long TIMEOUT = 10L; + private static final RangePartitioning.Range RANGE = + RangePartitioning.Range.newBuilder().setStart(1L).setInterval(2L).setEnd(10L).build(); + private static final RangePartitioning RANGE_PARTITIONING = + RangePartitioning.newBuilder().setField("IntegerField").setRange(RANGE).build(); private static final LoadJobConfiguration LOAD_CONFIGURATION_CSV = LoadJobConfiguration.newBuilder(TABLE_ID, SOURCE_URIS) .setCreateDisposition(CREATE_DISPOSITION) @@ -78,7 +82,9 @@ public class LoadJobConfigurationTest { .setClustering(CLUSTERING) .setLabels(LABELS) .setJobTimeoutMs(TIMEOUT) + .setRangePartitioning(RANGE_PARTITIONING) .build(); + private static final DatastoreBackupOptions BACKUP_OPTIONS = DatastoreBackupOptions.newBuilder() .setProjectionFields(ImmutableList.of("field_1", "field_2")) @@ -95,6 +101,7 @@ public class LoadJobConfigurationTest { .setAutodetect(AUTODETECT) .setLabels(LABELS) .setJobTimeoutMs(TIMEOUT) + .setRangePartitioning(RANGE_PARTITIONING) .build(); private static final LoadJobConfiguration LOAD_CONFIGURATION_AVRO = LoadJobConfiguration.newBuilder(TABLE_ID, SOURCE_URIS) @@ -112,6 +119,7 @@ public class LoadJobConfigurationTest { .setUseAvroLogicalTypes(USERAVROLOGICALTYPES) .setLabels(LABELS) .setJobTimeoutMs(TIMEOUT) + .setRangePartitioning(RANGE_PARTITIONING) .build(); @Test @@ -229,5 +237,6 @@ private void compareLoadJobConfiguration( assertEquals(expected.getUseAvroLogicalTypes(), value.getUseAvroLogicalTypes()); assertEquals(expected.getLabels(), value.getLabels()); assertEquals(expected.getJobTimeoutMs(), value.getJobTimeoutMs()); + assertEquals(expected.getRangePartitioning(), value.getRangePartitioning()); } } From 13436dcc155c8085bb68f91221bbdd94e9af174c Mon Sep 17 00:00:00 2001 From: Praful Makani Date: Wed, 23 Oct 2019 16:33:10 +0530 Subject: [PATCH 2/3] add range partitioning to tables, load job and query job --- .../cloud/bigquery/LoadJobConfiguration.java | 2 +- .../cloud/bigquery/QueryJobConfiguration.java | 33 +++++- .../bigquery/StandardTableDefinition.java | 19 ++++ .../bigquery/QueryJobConfigurationTest.java | 7 ++ .../cloud/bigquery/it/ITBigQueryTest.java | 104 ++++++++++++++++-- 5 files changed, 151 insertions(+), 14 deletions(-) diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java index ff537d98a3a1..c68e3f3b9f56 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java @@ -447,7 +447,7 @@ public Long getJobTimeoutMs() { return jobTimeoutMs; } - /** Returns the range partitioning specification for this table */ + /** Returns the range partitioning specification for the table */ public RangePartitioning getRangePartitioning() { return rangePartitioning; } diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryJobConfiguration.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryJobConfiguration.java index a49702e8c30a..2c1a9a43bae1 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryJobConfiguration.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryJobConfiguration.java @@ -67,6 +67,7 @@ public final class QueryJobConfiguration extends JobConfiguration { private final Clustering clustering; private final Long jobTimeoutMs; private final Map labels; + private final RangePartitioning rangePartitioning; /** * Priority levels for a query. If not specified the priority is assumed to be {@link @@ -114,6 +115,7 @@ public static final class Builder private Clustering clustering; private Long jobTimeoutMs; private Map labels; + private RangePartitioning rangePartitioning; private Builder() { super(Type.QUERY); @@ -144,6 +146,7 @@ private Builder(QueryJobConfiguration jobConfiguration) { this.clustering = jobConfiguration.clustering; this.jobTimeoutMs = jobConfiguration.jobTimeoutMs; this.labels = jobConfiguration.labels; + this.rangePartitioning = jobConfiguration.rangePartitioning; } private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) { @@ -233,6 +236,10 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur if (configurationPb.getLabels() != null) { this.labels = configurationPb.getLabels(); } + if (queryConfigurationPb.getRangePartitioning() != null) { + this.rangePartitioning = + RangePartitioning.fromPb(queryConfigurationPb.getRangePartitioning()); + } } /** Sets the BigQuery SQL query to execute. */ @@ -561,6 +568,17 @@ public Builder setLabels(Map labels) { return this; } + /** + * Range partitioning specification for this table. Only one of timePartitioning and + * rangePartitioning should be specified. + * + * @param rangePartitioning rangePartitioning or {@code null} for none + */ + public Builder setRangePartitioning(RangePartitioning rangePartitioning) { + this.rangePartitioning = rangePartitioning; + return this; + } + public QueryJobConfiguration build() { return new QueryJobConfiguration(this); } @@ -600,6 +618,7 @@ private QueryJobConfiguration(Builder builder) { this.clustering = builder.clustering; this.jobTimeoutMs = builder.jobTimeoutMs; this.labels = builder.labels; + this.rangePartitioning = builder.rangePartitioning; } /** @@ -779,6 +798,11 @@ public Map getLabels() { return labels; } + /** Returns the range partitioning specification for the table */ + public RangePartitioning getRangePartitioning() { + return rangePartitioning; + } + @Override public Builder toBuilder() { return new Builder(this); @@ -809,7 +833,8 @@ ToStringHelper toStringHelper() { .add("timePartitioning", timePartitioning) .add("clustering", clustering) .add("jobTimeoutMs", jobTimeoutMs) - .add("labels", labels); + .add("labels", labels) + .add("rangePartitioning", rangePartitioning); } @Override @@ -843,7 +868,8 @@ public int hashCode() { timePartitioning, clustering, jobTimeoutMs, - labels); + labels, + rangePartitioning); } @Override @@ -939,6 +965,9 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() { if (labels != null) { configurationPb.setLabels(labels); } + if (rangePartitioning != null) { + queryConfigurationPb.setRangePartitioning(rangePartitioning.toPb()); + } configurationPb.setQuery(queryConfigurationPb); return configurationPb; } diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardTableDefinition.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardTableDefinition.java index 9b2ae343875f..64f911d90c13 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardTableDefinition.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardTableDefinition.java @@ -149,6 +149,12 @@ public abstract static class Builder */ public abstract Builder setTimePartitioning(TimePartitioning timePartitioning); + /** + * Sets the range partitioning configuration for the table. Only one of timePartitioning and + * rangePartitioning should be specified. + */ + public abstract Builder setRangePartitioning(RangePartitioning rangePartitioning); + /** * Set the clustering configuration for the table. If not set, the table is not clustered. * Clustering is only available for partitioned tables. @@ -201,6 +207,13 @@ public abstract static class Builder @Nullable public abstract TimePartitioning getTimePartitioning(); + /** + * Returns the range partitioning configuration for this table. If {@code null}, the table is not + * range-partitioned. + */ + @Nullable + public abstract RangePartitioning getRangePartitioning(); + /** * Returns the clustering configuration for this table. If {@code null}, the table is not * clustered. @@ -240,6 +253,9 @@ Table toPb() { if (getTimePartitioning() != null) { tablePb.setTimePartitioning(getTimePartitioning().toPb()); } + if (getRangePartitioning() != null) { + tablePb.setRangePartitioning(getRangePartitioning().toPb()); + } if (getClustering() != null) { tablePb.setClustering(getClustering().toPb()); } @@ -258,6 +274,9 @@ static StandardTableDefinition fromPb(Table tablePb) { if (tablePb.getTimePartitioning() != null) { builder.setTimePartitioning(TimePartitioning.fromPb(tablePb.getTimePartitioning())); } + if (tablePb.getRangePartitioning() != null) { + builder.setRangePartitioning(RangePartitioning.fromPb(tablePb.getRangePartitioning())); + } if (tablePb.getClustering() != null) { builder.setClustering(Clustering.fromPb(tablePb.getClustering())); } diff --git a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryJobConfigurationTest.java b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryJobConfigurationTest.java index 5fdd9ab14891..b31312a345eb 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryJobConfigurationTest.java +++ b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryJobConfigurationTest.java @@ -86,6 +86,10 @@ public class QueryJobConfigurationTest { private static final Long TIMEOUT = 10L; private static final Map LABELS = ImmutableMap.of("test-job-name", "test-query-job"); + private static final RangePartitioning.Range RANGE = + RangePartitioning.Range.newBuilder().setStart(1L).setInterval(2L).setEnd(10L).build(); + private static final RangePartitioning RANGE_PARTITIONING = + RangePartitioning.newBuilder().setField("IntegerField").setRange(RANGE).build(); private static final QueryJobConfiguration QUERY_JOB_CONFIGURATION = QueryJobConfiguration.newBuilder(QUERY) .setUseQueryCache(USE_QUERY_CACHE) @@ -107,6 +111,7 @@ public class QueryJobConfigurationTest { .setClustering(CLUSTERING) .setJobTimeoutMs(TIMEOUT) .setLabels(LABELS) + .setRangePartitioning(RANGE_PARTITIONING) .build(); @Test @@ -140,6 +145,7 @@ public void testToPbAndFromPb() { assertNull(QUERY_JOB_CONFIGURATION.toPb().getLoad()); assertNotNull(QUERY_JOB_CONFIGURATION.getJobTimeoutMs()); assertNotNull(QUERY_JOB_CONFIGURATION.getLabels()); + assertNotNull(QUERY_JOB_CONFIGURATION.getRangePartitioning()); compareQueryJobConfiguration( QUERY_JOB_CONFIGURATION, QueryJobConfiguration.fromPb(QUERY_JOB_CONFIGURATION.toPb())); QueryJobConfiguration job = QueryJobConfiguration.of(QUERY); @@ -197,5 +203,6 @@ private void compareQueryJobConfiguration( assertEquals(expected.getClustering(), value.getClustering()); assertEquals(expected.getJobTimeoutMs(), value.getJobTimeoutMs()); assertEquals(expected.getLabels(), value.getLabels()); + assertEquals(expected.getRangePartitioning(), value.getRangePartitioning()); } } diff --git a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index e84d9b5ec3b4..7eb61773ff7e 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -67,6 +67,7 @@ import com.google.cloud.bigquery.ModelInfo; import com.google.cloud.bigquery.QueryJobConfiguration; import com.google.cloud.bigquery.QueryParameterValue; +import com.google.cloud.bigquery.RangePartitioning; import com.google.cloud.bigquery.Routine; import com.google.cloud.bigquery.RoutineArgument; import com.google.cloud.bigquery.RoutineId; @@ -215,6 +216,10 @@ public class ITBigQueryTest { Field.newBuilder("BooleanField", LegacySQLTypeName.BOOLEAN) .setMode(Field.Mode.NULLABLE) .build()); + private static final RangePartitioning.Range RANGE = + RangePartitioning.Range.newBuilder().setStart(1L).setInterval(2L).setEnd(20L).build(); + private static final RangePartitioning RANGE_PARTITIONING = + RangePartitioning.newBuilder().setField("IntegerField").setRange(RANGE).build(); private static final String LOAD_FILE = "load.csv"; private static final String JSON_LOAD_FILE = "load.json"; private static final String EXTRACT_FILE = "extract.csv"; @@ -453,6 +458,30 @@ public void testGetNonExistingTable() { assertNull(bigquery.getTable(DATASET, "test_get_non_existing_table")); } + @Test + public void testCreateTableWithRangePartitioning() { + String tableName = "test_create_table_rangepartitioning"; + TableId tableId = TableId.of(DATASET, tableName); + try { + StandardTableDefinition tableDefinition = + StandardTableDefinition.newBuilder() + .setSchema(TABLE_SCHEMA) + .setRangePartitioning(RANGE_PARTITIONING) + .build(); + Table createdTable = bigquery.create(TableInfo.of(tableId, tableDefinition)); + assertNotNull(createdTable); + Table remoteTable = bigquery.getTable(DATASET, tableName); + assertEquals( + RANGE, + remoteTable.getDefinition().getRangePartitioning().getRange()); + assertEquals( + RANGE_PARTITIONING, + remoteTable.getDefinition().getRangePartitioning()); + } finally { + bigquery.delete(tableId); + } + } + @Test public void testCreateAndGetTable() { String tableName = "test_create_and_get_table"; @@ -1546,17 +1575,70 @@ public void testQueryJobWithLabels() throws InterruptedException, TimeoutExcepti String query = "SELECT TimestampField, StringField, BooleanField FROM " + TABLE_ID.getTable(); Map labels = ImmutableMap.of("test-job-name", "test-query-job"); TableId destinationTable = TableId.of(DATASET, tableName); - QueryJobConfiguration configuration = - QueryJobConfiguration.newBuilder(query) - .setDefaultDataset(DatasetId.of(DATASET)) - .setDestinationTable(destinationTable) - .setLabels(labels) - .build(); - Job remoteJob = bigquery.create(JobInfo.of(configuration)); - remoteJob = remoteJob.waitFor(); - assertNull(remoteJob.getStatus().getError()); - QueryJobConfiguration queryJobConfiguration = remoteJob.getConfiguration(); - assertEquals(labels, queryJobConfiguration.getLabels()); + try { + QueryJobConfiguration configuration = + QueryJobConfiguration.newBuilder(query) + .setDefaultDataset(DatasetId.of(DATASET)) + .setDestinationTable(destinationTable) + .setLabels(labels) + .build(); + Job remoteJob = bigquery.create(JobInfo.of(configuration)); + remoteJob = remoteJob.waitFor(); + assertNull(remoteJob.getStatus().getError()); + QueryJobConfiguration queryJobConfiguration = remoteJob.getConfiguration(); + assertEquals(labels, queryJobConfiguration.getLabels()); + } finally { + bigquery.delete(destinationTable); + } + } + + @Test + public void testQueryJobWithRangePartitioning() throws InterruptedException { + String tableName = "test_query_job_table_rangepartitioning"; + String query = + "SELECT IntegerField, TimestampField, StringField, BooleanField FROM " + + TABLE_ID.getTable(); + TableId destinationTable = TableId.of(DATASET, tableName); + try { + QueryJobConfiguration configuration = + QueryJobConfiguration.newBuilder(query) + .setDefaultDataset(DatasetId.of(DATASET)) + .setDestinationTable(destinationTable) + .setRangePartitioning(RANGE_PARTITIONING) + .build(); + Job remoteJob = bigquery.create(JobInfo.of(configuration)); + remoteJob = remoteJob.waitFor(); + assertNull(remoteJob.getStatus().getError()); + QueryJobConfiguration queryJobConfiguration = remoteJob.getConfiguration(); + assertEquals(RANGE, queryJobConfiguration.getRangePartitioning().getRange()); + assertEquals(RANGE_PARTITIONING, queryJobConfiguration.getRangePartitioning()); + } finally { + bigquery.delete(destinationTable); + } + } + + @Test + public void testLoadJobWithRangePartitioning() throws InterruptedException { + String tableName = "test_load_job_table_rangepartitioning"; + TableId destinationTable = TableId.of(DATASET, tableName); + try { + LoadJobConfiguration configuration = + LoadJobConfiguration.newBuilder( + TABLE_ID, "gs://" + BUCKET + "/" + JSON_LOAD_FILE, FormatOptions.json()) + .setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) + .setSchema(TABLE_SCHEMA) + .setRangePartitioning(RANGE_PARTITIONING) + .setDestinationTable(destinationTable) + .build(); + Job job = bigquery.create(JobInfo.of(configuration)); + job = job.waitFor(); + assertNull(job.getStatus().getError()); + LoadJobConfiguration loadJobConfiguration = job.getConfiguration(); + assertEquals(RANGE, loadJobConfiguration.getRangePartitioning().getRange()); + assertEquals(RANGE_PARTITIONING, loadJobConfiguration.getRangePartitioning()); + } finally { + bigquery.delete(destinationTable); + } } @Test From ccaa9c6c3f6b92bf6e4e73d9f3ac849d1f8e1798 Mon Sep 17 00:00:00 2001 From: Praful Makani Date: Wed, 6 Nov 2019 09:57:51 +0530 Subject: [PATCH 3/3] fix javadoc --- .../main/java/com/google/cloud/bigquery/RangePartitioning.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/RangePartitioning.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/RangePartitioning.java index 54529d0bad54..a8b1850766ce 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/RangePartitioning.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/RangePartitioning.java @@ -188,7 +188,7 @@ private RangePartitioning( this.range = Range.fromPb(rangePartitioning.getRange()); } - /** Returns the field of range partitioning. */ + /** Returns the range partitioning field. */ public String getField() { return field; }