From c0af159ef5d888b954c9626b371387e59ac19acb Mon Sep 17 00:00:00 2001 From: stephwang Date: Fri, 17 Jan 2020 11:48:02 -0500 Subject: [PATCH 01/13] feat: add new sample - Tables: Relax column query append --- .../com/example/bigquery/RelaxTableQuery.java | 99 +++++++++++++++++++ .../example/bigquery/RelaxTableQueryIT.java | 87 ++++++++++++++++ 2 files changed, 186 insertions(+) create mode 100644 samples/src/main/java/com/example/bigquery/RelaxTableQuery.java create mode 100644 samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java diff --git a/samples/src/main/java/com/example/bigquery/RelaxTableQuery.java b/samples/src/main/java/com/example/bigquery/RelaxTableQuery.java new file mode 100644 index 000000000..cc4b5e7f7 --- /dev/null +++ b/samples/src/main/java/com/example/bigquery/RelaxTableQuery.java @@ -0,0 +1,99 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.bigquery; + +// [START bigquery_relax_column_query_append] +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQueryException; +import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.Job; +import com.google.cloud.bigquery.JobId; +import com.google.cloud.bigquery.JobInfo; +import com.google.cloud.bigquery.JobInfo.SchemaUpdateOption; +import com.google.cloud.bigquery.JobInfo.WriteDisposition; +import com.google.cloud.bigquery.QueryJobConfiguration; +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.TableResult; +import com.google.common.collect.ImmutableList; +import java.util.UUID; + +public class RelaxTableQuery { + + public static void runRelaxTableQuery() { + // TODO(developer): Replace these variables before running the sample. + String datasetName = "MY_DATASET_NAME"; + String tableName = "MY_TABLE_NAME"; + relaxTableQuery(datasetName, tableName); + } + + public static void relaxTableQuery(String datasetName, String tableName) { + try { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + + TableId tableId = TableId.of(datasetName, tableName); + String query = + "SELECT " + + "word " + // TODO(developer): Replace projectId.dataset.tableName with yours + + "FROM `java-docs-samples-testing.bigquery_test_dataset.RELAX_TABLE_QUERY_TEST` " + + "WHERE word like '%is%' "; + + QueryJobConfiguration queryConfig = + QueryJobConfiguration.newBuilder(query) + // Use standard SQL syntax for queries. + // See: https://cloud.google.com/bigquery/sql-reference/ + .setUseLegacySql(false) + .setSchemaUpdateOptions(ImmutableList.of(SchemaUpdateOption.ALLOW_FIELD_RELAXATION)) + .setWriteDisposition(WriteDisposition.WRITE_APPEND) + .setDestinationTable(tableId) + .build(); + + // Create a job ID so that we can safely retry. + JobId jobId = JobId.of(UUID.randomUUID().toString()); + Job queryJob = bigquery.create(JobInfo.newBuilder(queryConfig).setJobId(jobId).build()); + + queryJob = queryJob.waitFor(); + + // Check for errors + if (queryJob == null) { + throw new RuntimeException("Job no longer exists"); + } else if (queryJob.getStatus().getError() != null) { + // You can also look at queryJob.getStatus().getExecutionErrors() for all + // errors, not just the latest one. + throw new RuntimeException(queryJob.getStatus().getError().toString()); + } + + // Get the results. + TableResult results = queryJob.getQueryResults(); + + // Print all pages of the results. + results + .iterateAll() + .forEach( + rows -> { + rows.forEach(row -> System.out.println("row: " + row.toString())); + }); + + System.out.println("Successfully relaxed all columns in destination table during query job"); + } catch (BigQueryException | InterruptedException e) { + System.out.println("Columns not relaxed during query job \n" + e.toString()); + } + } +} +// [END bigquery_relax_column_query_append] diff --git a/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java b/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java new file mode 100644 index 000000000..2e8fbedc7 --- /dev/null +++ b/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java @@ -0,0 +1,87 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.bigquery; + +import static com.google.common.truth.Truth.assertThat; +import static junit.framework.TestCase.assertNotNull; +import static org.junit.Assert.*; + +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.LegacySQLTypeName; +import com.google.cloud.bigquery.Schema; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class RelaxTableQueryIT { + private ByteArrayOutputStream bout; + private PrintStream out; + + private static final String BIGQUERY_DATASET_NAME = System.getenv("BIGQUERY_DATASET_NAME"); + + private static void requireEnvVar(String varName) { + assertNotNull( + "Environment variable " + varName + " is required to perform these tests.", + System.getenv(varName)); + } + + @BeforeClass + public static void checkRequirements() { + requireEnvVar("BIGQUERY_DATASET_NAME"); + } + + @Before + public void setUp() throws Exception { + bout = new ByteArrayOutputStream(); + out = new PrintStream(bout); + System.setOut(out); + } + + @After + public void tearDown() { + System.setOut(null); + } + + @Test + public void testRelaxTableQuery() { + String tableName = "RELAX_TABLE_QUERY_TEST"; + Schema originalSchema = + Schema.of( + Field.newBuilder("word", LegacySQLTypeName.STRING).setMode(Field.Mode.REQUIRED).build(), + Field.newBuilder("word_count", LegacySQLTypeName.STRING) + .setMode(Field.Mode.REQUIRED) + .build(), + Field.newBuilder("corpus", LegacySQLTypeName.STRING) + .setMode(Field.Mode.REQUIRED) + .build(), + Field.newBuilder("corpus_date", LegacySQLTypeName.STRING) + .setMode(Field.Mode.REQUIRED) + .build()); + + CreateTable.createTable(BIGQUERY_DATASET_NAME, tableName, originalSchema); + + RelaxTableQuery.relaxTableQuery(BIGQUERY_DATASET_NAME, tableName); + assertThat(bout.toString()) + .contains("Successfully relaxed all columns in destination table during query job"); + + // Clean up + DeleteTable.deleteTable(BIGQUERY_DATASET_NAME, tableName); + } +} From 4ff1f36e7a152edf949cd4b729b03349558558b2 Mon Sep 17 00:00:00 2001 From: stephwang Date: Fri, 17 Jan 2020 12:01:00 -0500 Subject: [PATCH 02/13] nit --- .../src/test/java/com/example/bigquery/RelaxTableQueryIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java b/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java index 2e8fbedc7..a9049bbe0 100644 --- a/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java +++ b/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java @@ -18,7 +18,6 @@ import static com.google.common.truth.Truth.assertThat; import static junit.framework.TestCase.assertNotNull; -import static org.junit.Assert.*; import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.LegacySQLTypeName; From 1797825105c9e7db0fd9d479e6956bfae04d83a7 Mon Sep 17 00:00:00 2001 From: stephwang Date: Tue, 21 Jan 2020 13:17:53 -0500 Subject: [PATCH 03/13] update based on comments --- .../com/example/bigquery/RelaxTableQuery.java | 16 ++++++++-------- .../com/example/bigquery/RelaxTableQueryIT.java | 8 +++++++- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/samples/src/main/java/com/example/bigquery/RelaxTableQuery.java b/samples/src/main/java/com/example/bigquery/RelaxTableQuery.java index cc4b5e7f7..9a5a5fa45 100644 --- a/samples/src/main/java/com/example/bigquery/RelaxTableQuery.java +++ b/samples/src/main/java/com/example/bigquery/RelaxTableQuery.java @@ -37,22 +37,22 @@ public static void runRelaxTableQuery() { // TODO(developer): Replace these variables before running the sample. String datasetName = "MY_DATASET_NAME"; String tableName = "MY_TABLE_NAME"; - relaxTableQuery(datasetName, tableName); + String query = + "SELECT " + + "word " + // TODO(developer): Replace projectId.dataset.tableName with yours + + "FROM `java-docs-samples-testing.bigquery_test_dataset.RELAX_TABLE_QUERY_TEST` " + + "WHERE word like '%is%' "; + relaxTableQuery(datasetName, tableName, query); } - public static void relaxTableQuery(String datasetName, String tableName) { + public static void relaxTableQuery(String datasetName, String tableName, String query) { try { // Initialize client that will be used to send requests. This client only needs to be created // once, and can be reused for multiple requests. BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); TableId tableId = TableId.of(datasetName, tableName); - String query = - "SELECT " - + "word " - // TODO(developer): Replace projectId.dataset.tableName with yours - + "FROM `java-docs-samples-testing.bigquery_test_dataset.RELAX_TABLE_QUERY_TEST` " - + "WHERE word like '%is%' "; QueryJobConfiguration queryConfig = QueryJobConfiguration.newBuilder(query) diff --git a/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java b/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java index a9049bbe0..f8d66777a 100644 --- a/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java +++ b/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java @@ -74,9 +74,15 @@ public void testRelaxTableQuery() { .setMode(Field.Mode.REQUIRED) .build()); + String query = + "SELECT " + + "word " + + "FROM `java-docs-samples-testing.bigquery_test_dataset.RELAX_TABLE_QUERY_TEST` " + + "WHERE word like '%is%' "; + CreateTable.createTable(BIGQUERY_DATASET_NAME, tableName, originalSchema); - RelaxTableQuery.relaxTableQuery(BIGQUERY_DATASET_NAME, tableName); + RelaxTableQuery.relaxTableQuery(BIGQUERY_DATASET_NAME, tableName, query); assertThat(bout.toString()) .contains("Successfully relaxed all columns in destination table during query job"); From c49b78b50527918dcdb64fbf11feb5d0af91a7e7 Mon Sep 17 00:00:00 2001 From: stephwang Date: Wed, 22 Jan 2020 11:50:45 -0500 Subject: [PATCH 04/13] update exception handling --- .../main/java/com/example/bigquery/RelaxTableQuery.java | 9 +++++---- .../java/com/example/bigquery/RelaxTableQueryIT.java | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/samples/src/main/java/com/example/bigquery/RelaxTableQuery.java b/samples/src/main/java/com/example/bigquery/RelaxTableQuery.java index 9a5a5fa45..4a4403419 100644 --- a/samples/src/main/java/com/example/bigquery/RelaxTableQuery.java +++ b/samples/src/main/java/com/example/bigquery/RelaxTableQuery.java @@ -33,7 +33,7 @@ public class RelaxTableQuery { - public static void runRelaxTableQuery() { + public static void runRelaxTableQuery() throws Exception { // TODO(developer): Replace these variables before running the sample. String datasetName = "MY_DATASET_NAME"; String tableName = "MY_TABLE_NAME"; @@ -46,7 +46,8 @@ public static void runRelaxTableQuery() { relaxTableQuery(datasetName, tableName, query); } - public static void relaxTableQuery(String datasetName, String tableName, String query) { + public static void relaxTableQuery(String datasetName, String tableName, String query) + throws Exception { try { // Initialize client that will be used to send requests. This client only needs to be created // once, and can be reused for multiple requests. @@ -72,11 +73,11 @@ public static void relaxTableQuery(String datasetName, String tableName, String // Check for errors if (queryJob == null) { - throw new RuntimeException("Job no longer exists"); + throw new Exception("Job no longer exists"); } else if (queryJob.getStatus().getError() != null) { // You can also look at queryJob.getStatus().getExecutionErrors() for all // errors, not just the latest one. - throw new RuntimeException(queryJob.getStatus().getError().toString()); + throw new Exception(queryJob.getStatus().getError().toString()); } // Get the results. diff --git a/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java b/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java index f8d66777a..e70ee7740 100644 --- a/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java +++ b/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java @@ -59,7 +59,7 @@ public void tearDown() { } @Test - public void testRelaxTableQuery() { + public void testRelaxTableQuery() throws Exception { String tableName = "RELAX_TABLE_QUERY_TEST"; Schema originalSchema = Schema.of( From a1393e4b3244a3c5db9e6a361909ae6368e6be48 Mon Sep 17 00:00:00 2001 From: stephwang Date: Thu, 23 Jan 2020 12:18:30 -0500 Subject: [PATCH 05/13] refactor --- .../com/example/bigquery/RelaxTableQuery.java | 15 +++++++-------- .../com/example/bigquery/RelaxTableQueryIT.java | 10 +++------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/samples/src/main/java/com/example/bigquery/RelaxTableQuery.java b/samples/src/main/java/com/example/bigquery/RelaxTableQuery.java index 4a4403419..d4f515325 100644 --- a/samples/src/main/java/com/example/bigquery/RelaxTableQuery.java +++ b/samples/src/main/java/com/example/bigquery/RelaxTableQuery.java @@ -35,18 +35,13 @@ public class RelaxTableQuery { public static void runRelaxTableQuery() throws Exception { // TODO(developer): Replace these variables before running the sample. + String projectId = "MY_PROJECT_ID"; String datasetName = "MY_DATASET_NAME"; String tableName = "MY_TABLE_NAME"; - String query = - "SELECT " - + "word " - // TODO(developer): Replace projectId.dataset.tableName with yours - + "FROM `java-docs-samples-testing.bigquery_test_dataset.RELAX_TABLE_QUERY_TEST` " - + "WHERE word like '%is%' "; - relaxTableQuery(datasetName, tableName, query); + relaxTableQuery(projectId, datasetName, tableName); } - public static void relaxTableQuery(String datasetName, String tableName, String query) + public static void relaxTableQuery(String projectId, String datasetName, String tableName) throws Exception { try { // Initialize client that will be used to send requests. This client only needs to be created @@ -55,6 +50,10 @@ public static void relaxTableQuery(String datasetName, String tableName, String TableId tableId = TableId.of(datasetName, tableName); + String sourceTable = "`" + projectId + "." + datasetName + "." + tableName + "`"; + String query = + "SELECT " + "word\n" + "FROM " + sourceTable + "\n" + "WHERE word like '%is%' "; + QueryJobConfiguration queryConfig = QueryJobConfiguration.newBuilder(query) // Use standard SQL syntax for queries. diff --git a/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java b/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java index e70ee7740..af7391456 100644 --- a/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java +++ b/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java @@ -33,6 +33,7 @@ public class RelaxTableQueryIT { private ByteArrayOutputStream bout; private PrintStream out; + private static final String BIGQUERY_PROJECT_ID = System.getenv("BIGQUERY_PROJECT_ID"); private static final String BIGQUERY_DATASET_NAME = System.getenv("BIGQUERY_DATASET_NAME"); private static void requireEnvVar(String varName) { @@ -43,6 +44,7 @@ private static void requireEnvVar(String varName) { @BeforeClass public static void checkRequirements() { + requireEnvVar("BIGQUERY_PROJECT_ID"); requireEnvVar("BIGQUERY_DATASET_NAME"); } @@ -74,15 +76,9 @@ public void testRelaxTableQuery() throws Exception { .setMode(Field.Mode.REQUIRED) .build()); - String query = - "SELECT " - + "word " - + "FROM `java-docs-samples-testing.bigquery_test_dataset.RELAX_TABLE_QUERY_TEST` " - + "WHERE word like '%is%' "; - CreateTable.createTable(BIGQUERY_DATASET_NAME, tableName, originalSchema); - RelaxTableQuery.relaxTableQuery(BIGQUERY_DATASET_NAME, tableName, query); + RelaxTableQuery.relaxTableQuery(BIGQUERY_PROJECT_ID, BIGQUERY_DATASET_NAME, tableName); assertThat(bout.toString()) .contains("Successfully relaxed all columns in destination table during query job"); From 3d35c13d828c1c4b7d396b78df8bc5f37742ef1c Mon Sep 17 00:00:00 2001 From: stephwang Date: Thu, 23 Jan 2020 14:26:16 -0500 Subject: [PATCH 06/13] update base on comments --- .../main/java/com/example/bigquery/RelaxTableQuery.java | 1 + .../test/java/com/example/bigquery/RelaxTableQueryIT.java | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/samples/src/main/java/com/example/bigquery/RelaxTableQuery.java b/samples/src/main/java/com/example/bigquery/RelaxTableQuery.java index d4f515325..ddd177cb4 100644 --- a/samples/src/main/java/com/example/bigquery/RelaxTableQuery.java +++ b/samples/src/main/java/com/example/bigquery/RelaxTableQuery.java @@ -41,6 +41,7 @@ public static void runRelaxTableQuery() throws Exception { relaxTableQuery(projectId, datasetName, tableName); } + // To relax all columns in a destination table when you append data to it during a query job public static void relaxTableQuery(String projectId, String datasetName, String tableName) throws Exception { try { diff --git a/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java b/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java index af7391456..8922b800a 100644 --- a/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java +++ b/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java @@ -24,6 +24,7 @@ import com.google.cloud.bigquery.Schema; import java.io.ByteArrayOutputStream; import java.io.PrintStream; +import java.util.UUID; import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; @@ -32,6 +33,7 @@ public class RelaxTableQueryIT { private ByteArrayOutputStream bout; private PrintStream out; + private String tableName; private static final String BIGQUERY_PROJECT_ID = System.getenv("BIGQUERY_PROJECT_ID"); private static final String BIGQUERY_DATASET_NAME = System.getenv("BIGQUERY_DATASET_NAME"); @@ -52,17 +54,18 @@ public static void checkRequirements() { public void setUp() throws Exception { bout = new ByteArrayOutputStream(); out = new PrintStream(bout); + tableName = "RELAX_TABLE_QUERY_TEST" + UUID.randomUUID().toString().replace('-', '_'); System.setOut(out); } @After public void tearDown() { + DeleteTable.deleteTable(BIGQUERY_DATASET_NAME, tableName); System.setOut(null); } @Test public void testRelaxTableQuery() throws Exception { - String tableName = "RELAX_TABLE_QUERY_TEST"; Schema originalSchema = Schema.of( Field.newBuilder("word", LegacySQLTypeName.STRING).setMode(Field.Mode.REQUIRED).build(), @@ -81,8 +84,5 @@ public void testRelaxTableQuery() throws Exception { RelaxTableQuery.relaxTableQuery(BIGQUERY_PROJECT_ID, BIGQUERY_DATASET_NAME, tableName); assertThat(bout.toString()) .contains("Successfully relaxed all columns in destination table during query job"); - - // Clean up - DeleteTable.deleteTable(BIGQUERY_DATASET_NAME, tableName); } } From d5c035ecb4a9bce4c64b7cbc8ce405edd7d45fce Mon Sep 17 00:00:00 2001 From: stephwang Date: Thu, 23 Jan 2020 14:52:02 -0500 Subject: [PATCH 07/13] code refactoring --- .../example/bigquery/RelaxTableQueryIT.java | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java b/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java index 8922b800a..0534c9f4b 100644 --- a/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java +++ b/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java @@ -34,6 +34,7 @@ public class RelaxTableQueryIT { private ByteArrayOutputStream bout; private PrintStream out; private String tableName; + private Schema originalSchema; private static final String BIGQUERY_PROJECT_ID = System.getenv("BIGQUERY_PROJECT_ID"); private static final String BIGQUERY_DATASET_NAME = System.getenv("BIGQUERY_DATASET_NAME"); @@ -54,19 +55,9 @@ public static void checkRequirements() { public void setUp() throws Exception { bout = new ByteArrayOutputStream(); out = new PrintStream(bout); - tableName = "RELAX_TABLE_QUERY_TEST" + UUID.randomUUID().toString().replace('-', '_'); - System.setOut(out); - } - - @After - public void tearDown() { - DeleteTable.deleteTable(BIGQUERY_DATASET_NAME, tableName); - System.setOut(null); - } - - @Test - public void testRelaxTableQuery() throws Exception { - Schema originalSchema = + tableName = + "RELAX_TABLE_QUERY_TEST" + UUID.randomUUID().toString().substring(0, 5).replace('-', '_'); + originalSchema = Schema.of( Field.newBuilder("word", LegacySQLTypeName.STRING).setMode(Field.Mode.REQUIRED).build(), Field.newBuilder("word_count", LegacySQLTypeName.STRING) @@ -78,9 +69,18 @@ public void testRelaxTableQuery() throws Exception { Field.newBuilder("corpus_date", LegacySQLTypeName.STRING) .setMode(Field.Mode.REQUIRED) .build()); - CreateTable.createTable(BIGQUERY_DATASET_NAME, tableName, originalSchema); + System.setOut(out); + } + @After + public void tearDown() { + DeleteTable.deleteTable(BIGQUERY_DATASET_NAME, tableName); + System.setOut(null); + } + + @Test + public void testRelaxTableQuery() throws Exception { RelaxTableQuery.relaxTableQuery(BIGQUERY_PROJECT_ID, BIGQUERY_DATASET_NAME, tableName); assertThat(bout.toString()) .contains("Successfully relaxed all columns in destination table during query job"); From 0c13ae926fc8793621d15fa52da5037c01d1aa4a Mon Sep 17 00:00:00 2001 From: stephwang Date: Thu, 23 Jan 2020 15:41:09 -0500 Subject: [PATCH 08/13] fix build errors --- .../example/bigquery/RelaxTableQueryIT.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java b/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java index 0534c9f4b..566003fff 100644 --- a/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java +++ b/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java @@ -33,11 +33,11 @@ public class RelaxTableQueryIT { private ByteArrayOutputStream bout; private PrintStream out; - private String tableName; - private Schema originalSchema; private static final String BIGQUERY_PROJECT_ID = System.getenv("BIGQUERY_PROJECT_ID"); private static final String BIGQUERY_DATASET_NAME = System.getenv("BIGQUERY_DATASET_NAME"); + private static final String TABLE_NAME = + "RELAX_TABLE_QUERY_TEST_" + UUID.randomUUID().toString().replace('-', '_'); private static void requireEnvVar(String varName) { assertNotNull( @@ -55,9 +55,10 @@ public static void checkRequirements() { public void setUp() throws Exception { bout = new ByteArrayOutputStream(); out = new PrintStream(bout); - tableName = - "RELAX_TABLE_QUERY_TEST" + UUID.randomUUID().toString().substring(0, 5).replace('-', '_'); - originalSchema = + + CreateTable.createTable( + BIGQUERY_DATASET_NAME, + TABLE_NAME, Schema.of( Field.newBuilder("word", LegacySQLTypeName.STRING).setMode(Field.Mode.REQUIRED).build(), Field.newBuilder("word_count", LegacySQLTypeName.STRING) @@ -68,20 +69,19 @@ public void setUp() throws Exception { .build(), Field.newBuilder("corpus_date", LegacySQLTypeName.STRING) .setMode(Field.Mode.REQUIRED) - .build()); - CreateTable.createTable(BIGQUERY_DATASET_NAME, tableName, originalSchema); + .build())); System.setOut(out); } @After public void tearDown() { - DeleteTable.deleteTable(BIGQUERY_DATASET_NAME, tableName); + DeleteTable.deleteTable(BIGQUERY_DATASET_NAME, TABLE_NAME); System.setOut(null); } @Test public void testRelaxTableQuery() throws Exception { - RelaxTableQuery.relaxTableQuery(BIGQUERY_PROJECT_ID, BIGQUERY_DATASET_NAME, tableName); + RelaxTableQuery.relaxTableQuery(BIGQUERY_PROJECT_ID, BIGQUERY_DATASET_NAME, TABLE_NAME); assertThat(bout.toString()) .contains("Successfully relaxed all columns in destination table during query job"); } From b022684f2ad77ecb5db81a0371f72fe7926911d2 Mon Sep 17 00:00:00 2001 From: stephwang Date: Thu, 23 Jan 2020 16:36:18 -0500 Subject: [PATCH 09/13] add debugging statements --- .../src/test/java/com/example/bigquery/RelaxTableQueryIT.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java b/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java index 566003fff..3f4c99cce 100644 --- a/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java +++ b/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java @@ -53,6 +53,9 @@ public static void checkRequirements() { @Before public void setUp() throws Exception { + System.out.println("BIGQUERY_DATASET_NAME: " + BIGQUERY_DATASET_NAME); + System.out.println("TABLE_NAME: " + TABLE_NAME); + bout = new ByteArrayOutputStream(); out = new PrintStream(bout); From 7d9118990f10af553733c209fd0855c898c91065 Mon Sep 17 00:00:00 2001 From: stephwang Date: Thu, 23 Jan 2020 16:43:54 -0500 Subject: [PATCH 10/13] remove unused exception --- .../src/test/java/com/example/bigquery/RelaxTableQueryIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java b/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java index 3f4c99cce..2236674e0 100644 --- a/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java +++ b/samples/src/test/java/com/example/bigquery/RelaxTableQueryIT.java @@ -52,7 +52,7 @@ public static void checkRequirements() { } @Before - public void setUp() throws Exception { + public void setUp() { System.out.println("BIGQUERY_DATASET_NAME: " + BIGQUERY_DATASET_NAME); System.out.println("TABLE_NAME: " + TABLE_NAME); From 3c0aab0f824fb9b73903dbac5ddd5108c45eeca8 Mon Sep 17 00:00:00 2001 From: stephwang Date: Thu, 23 Jan 2020 18:21:56 -0500 Subject: [PATCH 11/13] feat: new sample - Models: Get a model resource --- .../java/com/example/bigquery/GetModel.java | 52 +++++++++++++++ .../java/com/example/bigquery/GetModelIT.java | 66 +++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 samples/src/main/java/com/example/bigquery/GetModel.java create mode 100644 samples/src/test/java/com/example/bigquery/GetModelIT.java diff --git a/samples/src/main/java/com/example/bigquery/GetModel.java b/samples/src/main/java/com/example/bigquery/GetModel.java new file mode 100644 index 000000000..cc9a974fc --- /dev/null +++ b/samples/src/main/java/com/example/bigquery/GetModel.java @@ -0,0 +1,52 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.bigquery; + +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQueryException; +import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.Model; +import com.google.cloud.bigquery.ModelId; + +// [START bigquery_get_model] +public class GetModel { + + public static void runGetModel() { + // TODO(developer): Replace these variables before running the sample. + String datasetName = "MY_DATASET_NAME"; + String modelName = "MY_MODEL_ID"; + getModel(datasetName, modelName); + } + + public static void getModel(String datasetName, String modelName) { + try { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + + ModelId modelId = ModelId.of(datasetName, modelName); + Model model = bigquery.getModel(modelId); + System.out.println("Model: " + model.getDescription()); + + System.out.println("Successfully retrieved model"); + } catch (BigQueryException e) { + System.out.println("Cannot retrieve model \n" + e.toString()); + } + } + +} +// [END bigquery_get_model] \ No newline at end of file diff --git a/samples/src/test/java/com/example/bigquery/GetModelIT.java b/samples/src/test/java/com/example/bigquery/GetModelIT.java new file mode 100644 index 000000000..f856d070d --- /dev/null +++ b/samples/src/test/java/com/example/bigquery/GetModelIT.java @@ -0,0 +1,66 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.bigquery; + +import static com.google.common.truth.Truth.assertThat; +import static junit.framework.TestCase.assertNotNull; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class GetModelIT { + private ByteArrayOutputStream bout; + private PrintStream out; + + private static final String BIGQUERY_MODEL_TEST_PROJECT_ID = System.getenv("BIGQUERY_MODEL_TEST_PROJECT_ID"); + + private static void requireEnvVar(String varName) { + assertNotNull( + "Environment variable " + varName + " is required to perform these tests.", + System.getenv(varName)); + } + + @BeforeClass + public static void checkRequirements() { + requireEnvVar("BIGQUERY_MODEL_TEST_PROJECT_ID"); + } + + @Before + public void setUp() throws Exception { + bout = new ByteArrayOutputStream(); + out = new PrintStream(bout); + System.setOut(out); + } + + @After + public void tearDown() { + System.setOut(null); + } + + @Test + public void getModel() { + String datasetName = "samples"; + String modelName = "model"; + GetModel.getModel(datasetName, modelName); + assertThat(bout.toString()) + .contains("Successfully retrieved model"); + } +} \ No newline at end of file From 8fcb76dbf657633288ae0725d2320d5dfedb7276 Mon Sep 17 00:00:00 2001 From: stephwang Date: Thu, 23 Jan 2020 18:22:11 -0500 Subject: [PATCH 12/13] feat: new sample - Models: Get a model resource --- samples/src/main/java/com/example/bigquery/GetModel.java | 3 +-- .../src/main/java/com/example/bigquery/LoadParquet.java | 2 +- .../src/test/java/com/example/bigquery/GetModelIT.java | 8 ++++---- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/samples/src/main/java/com/example/bigquery/GetModel.java b/samples/src/main/java/com/example/bigquery/GetModel.java index cc9a974fc..e51b3fd83 100644 --- a/samples/src/main/java/com/example/bigquery/GetModel.java +++ b/samples/src/main/java/com/example/bigquery/GetModel.java @@ -47,6 +47,5 @@ public static void getModel(String datasetName, String modelName) { System.out.println("Cannot retrieve model \n" + e.toString()); } } - } -// [END bigquery_get_model] \ No newline at end of file +// [END bigquery_get_model] diff --git a/samples/src/main/java/com/example/bigquery/LoadParquet.java b/samples/src/main/java/com/example/bigquery/LoadParquet.java index ed04ba4dd..580b6183f 100644 --- a/samples/src/main/java/com/example/bigquery/LoadParquet.java +++ b/samples/src/main/java/com/example/bigquery/LoadParquet.java @@ -43,7 +43,7 @@ public static void loadParquet(String datasetName) { String sourceUri = "gs://cloud-samples-data/bigquery/us-states/us-states.parquet"; TableId tableId = TableId.of(datasetName, "us_states"); - + LoadJobConfiguration configuration = LoadJobConfiguration.builder(tableId, sourceUri) .setFormatOptions(FormatOptions.parquet()) diff --git a/samples/src/test/java/com/example/bigquery/GetModelIT.java b/samples/src/test/java/com/example/bigquery/GetModelIT.java index f856d070d..f70b522b3 100644 --- a/samples/src/test/java/com/example/bigquery/GetModelIT.java +++ b/samples/src/test/java/com/example/bigquery/GetModelIT.java @@ -30,7 +30,8 @@ public class GetModelIT { private ByteArrayOutputStream bout; private PrintStream out; - private static final String BIGQUERY_MODEL_TEST_PROJECT_ID = System.getenv("BIGQUERY_MODEL_TEST_PROJECT_ID"); + private static final String BIGQUERY_MODEL_TEST_PROJECT_ID = + System.getenv("BIGQUERY_MODEL_TEST_PROJECT_ID"); private static void requireEnvVar(String varName) { assertNotNull( @@ -60,7 +61,6 @@ public void getModel() { String datasetName = "samples"; String modelName = "model"; GetModel.getModel(datasetName, modelName); - assertThat(bout.toString()) - .contains("Successfully retrieved model"); + assertThat(bout.toString()).contains("Successfully retrieved model"); } -} \ No newline at end of file +} From 181635c72743ab9dc503b9e65b5c79d91e8309a7 Mon Sep 17 00:00:00 2001 From: stephwang Date: Mon, 27 Jan 2020 15:08:39 -0500 Subject: [PATCH 13/13] fix build error --- .../test/java/com/example/bigquery/GetModelIT.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/samples/src/test/java/com/example/bigquery/GetModelIT.java b/samples/src/test/java/com/example/bigquery/GetModelIT.java index f70b522b3..78d53ad7c 100644 --- a/samples/src/test/java/com/example/bigquery/GetModelIT.java +++ b/samples/src/test/java/com/example/bigquery/GetModelIT.java @@ -30,8 +30,8 @@ public class GetModelIT { private ByteArrayOutputStream bout; private PrintStream out; - private static final String BIGQUERY_MODEL_TEST_PROJECT_ID = - System.getenv("BIGQUERY_MODEL_TEST_PROJECT_ID"); + private static final String BIGQUERY_DATASET_NAME = System.getenv("BIGQUERY_DATASET_NAME"); + private static final String BIGQUERY_MODEL_NAME = System.getenv("BIGQUERY_MODEL_NAME"); private static void requireEnvVar(String varName) { assertNotNull( @@ -41,7 +41,8 @@ private static void requireEnvVar(String varName) { @BeforeClass public static void checkRequirements() { - requireEnvVar("BIGQUERY_MODEL_TEST_PROJECT_ID"); + requireEnvVar("BIGQUERY_DATASET_NAME"); + requireEnvVar("BIGQUERY_MODEL_NAME"); } @Before @@ -58,9 +59,7 @@ public void tearDown() { @Test public void getModel() { - String datasetName = "samples"; - String modelName = "model"; - GetModel.getModel(datasetName, modelName); + GetModel.getModel(BIGQUERY_DATASET_NAME, BIGQUERY_MODEL_NAME); assertThat(bout.toString()).contains("Successfully retrieved model"); } }