From 37410df0ecf8da2edc5e1f7df94c8290bd5e375d Mon Sep 17 00:00:00 2001 From: Mairbek Khadikov Date: Mon, 5 Jun 2017 12:29:02 -0700 Subject: [PATCH 1/3] Generate a random table name. Assume Spanner database exists. --- .../sdk/io/gcp/spanner/SpannerWriteIT.java | 35 +++++++++---------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java index 064c65eedcef..a03f0014c248 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java @@ -21,7 +21,6 @@ import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertThat; -import com.google.cloud.spanner.Database; import com.google.cloud.spanner.DatabaseAdminClient; import com.google.cloud.spanner.DatabaseClient; import com.google.cloud.spanner.DatabaseId; @@ -31,8 +30,10 @@ import com.google.cloud.spanner.Spanner; import com.google.cloud.spanner.SpannerOptions; import com.google.cloud.spanner.Statement; -import com.google.spanner.admin.database.v1.CreateDatabaseMetadata; import java.util.Collections; +import java.util.UUID; + +import com.google.spanner.admin.database.v1.UpdateDatabaseDdlMetadata; import org.apache.beam.sdk.io.GenerateSequence; import org.apache.beam.sdk.options.Default; import org.apache.beam.sdk.options.Description; @@ -49,6 +50,7 @@ import org.junit.runner.RunWith; import org.junit.runners.JUnit4; + /** End-to-end test of Cloud Spanner Sink. */ @RunWith(JUnit4.class) public class SpannerWriteIT { @@ -70,16 +72,11 @@ public interface SpannerTestPipelineOptions extends TestPipelineOptions { @Default.String("beam-testdb") String getDatabaseId(); void setDatabaseId(String value); - - @Description("Table name") - @Default.String("users") - String getTable(); - void setTable(String value); } private Spanner spanner; - private DatabaseAdminClient databaseAdminClient; private SpannerTestPipelineOptions options; + private final String tableName = generateTableName(); @Before public void setUp() throws Exception { @@ -88,29 +85,28 @@ public void setUp() throws Exception { spanner = SpannerOptions.newBuilder().setProjectId(options.getProjectId()).build().getService(); - databaseAdminClient = spanner.getDatabaseAdminClient(); + DatabaseAdminClient databaseAdminClient = spanner.getDatabaseAdminClient(); - // Delete database if exists. - databaseAdminClient.dropDatabase(options.getInstanceId(), options.getDatabaseId()); + databaseAdminClient.getDatabase(options.getInstanceId(), options.getDatabaseId()); - Operation op = - databaseAdminClient.createDatabase( + Operation op = + databaseAdminClient.updateDatabaseDdl( options.getInstanceId(), options.getDatabaseId(), Collections.singleton( "CREATE TABLE " - + options.getTable() + + tableName + " (" + " Key INT64," + " Value STRING(MAX)," - + ") PRIMARY KEY (Key)")); + + ") PRIMARY KEY (Key)"), null); op.waitFor(); } @Test public void testWrite() throws Exception { p.apply(GenerateSequence.from(0).to(100)) - .apply(ParDo.of(new GenerateMutations(options.getTable()))) + .apply(ParDo.of(new GenerateMutations(tableName))) .apply( SpannerIO.write() .withProjectId(options.getProjectId()) @@ -126,7 +122,7 @@ public void testWrite() throws Exception { ResultSet resultSet = databaseClient .singleUse() - .executeQuery(Statement.of("SELECT COUNT(*) FROM " + options.getTable())); + .executeQuery(Statement.of("SELECT COUNT(*) FROM " + tableName)); assertThat(resultSet.next(), is(true)); assertThat(resultSet.getLong(0), equalTo(100L)); assertThat(resultSet.next(), is(false)); @@ -134,7 +130,6 @@ public void testWrite() throws Exception { @After public void tearDown() throws Exception { - databaseAdminClient.dropDatabase(options.getInstanceId(), options.getDatabaseId()); spanner.closeAsync().get(); } @@ -156,4 +151,8 @@ public void processElement(ProcessContext c) { c.output(mutation); } } + + private static String generateTableName() { + return "test-table-" + UUID.randomUUID().toString(); + } } From 17359e6be7b1e88abbf7ca9b5c0188d726f22e2e Mon Sep 17 00:00:00 2001 From: Mairbek Khadikov Date: Mon, 5 Jun 2017 13:01:55 -0700 Subject: [PATCH 2/3] Actually it faster to create and drop databases than tables. --- .../sdk/io/gcp/spanner/SpannerWriteIT.java | 52 +++++++++++-------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java index a03f0014c248..3a0434edb89d 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java @@ -21,6 +21,7 @@ import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertThat; +import com.google.cloud.spanner.Database; import com.google.cloud.spanner.DatabaseAdminClient; import com.google.cloud.spanner.DatabaseClient; import com.google.cloud.spanner.DatabaseId; @@ -30,10 +31,10 @@ import com.google.cloud.spanner.Spanner; import com.google.cloud.spanner.SpannerOptions; import com.google.cloud.spanner.Statement; +import com.google.spanner.admin.database.v1.CreateDatabaseMetadata; import java.util.Collections; import java.util.UUID; -import com.google.spanner.admin.database.v1.UpdateDatabaseDdlMetadata; import org.apache.beam.sdk.io.GenerateSequence; import org.apache.beam.sdk.options.Default; import org.apache.beam.sdk.options.Description; @@ -50,7 +51,6 @@ import org.junit.runner.RunWith; import org.junit.runners.JUnit4; - /** End-to-end test of Cloud Spanner Sink. */ @RunWith(JUnit4.class) public class SpannerWriteIT { @@ -68,15 +68,21 @@ public interface SpannerTestPipelineOptions extends TestPipelineOptions { String getInstanceId(); void setInstanceId(String value); - @Description("Database ID to write to in Spanner") + @Description("Database ID prefix to write to in Spanner") @Default.String("beam-testdb") - String getDatabaseId(); - void setDatabaseId(String value); + String getDatabaseIdPrefix(); + void setDatabaseIdPrefix(String value); + + @Description("Table name") + @Default.String("users") + String getTable(); + void setTable(String value); } private Spanner spanner; + private DatabaseAdminClient databaseAdminClient; private SpannerTestPipelineOptions options; - private final String tableName = generateTableName(); + private String databaseName; @Before public void setUp() throws Exception { @@ -85,44 +91,51 @@ public void setUp() throws Exception { spanner = SpannerOptions.newBuilder().setProjectId(options.getProjectId()).build().getService(); - DatabaseAdminClient databaseAdminClient = spanner.getDatabaseAdminClient(); + databaseName = generateDatabaseName(); - databaseAdminClient.getDatabase(options.getInstanceId(), options.getDatabaseId()); + databaseAdminClient = spanner.getDatabaseAdminClient(); - Operation op = - databaseAdminClient.updateDatabaseDdl( + // Delete database if exists. + databaseAdminClient.dropDatabase(options.getInstanceId(), databaseName); + + Operation op = + databaseAdminClient.createDatabase( options.getInstanceId(), - options.getDatabaseId(), + databaseName, Collections.singleton( "CREATE TABLE " - + tableName + + options.getTable() + " (" + " Key INT64," + " Value STRING(MAX)," - + ") PRIMARY KEY (Key)"), null); + + ") PRIMARY KEY (Key)")); op.waitFor(); } + private String generateDatabaseName() { + return options.getDatabaseIdPrefix() + "-" + UUID.randomUUID().toString(); + } + @Test public void testWrite() throws Exception { p.apply(GenerateSequence.from(0).to(100)) - .apply(ParDo.of(new GenerateMutations(tableName))) + .apply(ParDo.of(new GenerateMutations(options.getTable()))) .apply( SpannerIO.write() .withProjectId(options.getProjectId()) .withInstanceId(options.getInstanceId()) - .withDatabaseId(options.getDatabaseId())); + .withDatabaseId(databaseName)); p.run(); DatabaseClient databaseClient = spanner.getDatabaseClient( DatabaseId.of( - options.getProjectId(), options.getInstanceId(), options.getDatabaseId())); + options.getProjectId(), options.getInstanceId(), databaseName)); ResultSet resultSet = databaseClient .singleUse() - .executeQuery(Statement.of("SELECT COUNT(*) FROM " + tableName)); + .executeQuery(Statement.of("SELECT COUNT(*) FROM " + options.getTable())); assertThat(resultSet.next(), is(true)); assertThat(resultSet.getLong(0), equalTo(100L)); assertThat(resultSet.next(), is(false)); @@ -130,6 +143,7 @@ public void testWrite() throws Exception { @After public void tearDown() throws Exception { + databaseAdminClient.dropDatabase(options.getInstanceId(), databaseName); spanner.closeAsync().get(); } @@ -151,8 +165,4 @@ public void processElement(ProcessContext c) { c.output(mutation); } } - - private static String generateTableName() { - return "test-table-" + UUID.randomUUID().toString(); - } } From fc2542a0e0f876d2c9337efea79bf83a614462f2 Mon Sep 17 00:00:00 2001 From: Mairbek Khadikov Date: Mon, 5 Jun 2017 13:16:12 -0700 Subject: [PATCH 3/3] Use RandomStringUtils --- .../beam/sdk/io/gcp/spanner/SpannerWriteIT.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java index 3a0434edb89d..8df224b76a08 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/spanner/SpannerWriteIT.java @@ -33,7 +33,6 @@ import com.google.cloud.spanner.Statement; import com.google.spanner.admin.database.v1.CreateDatabaseMetadata; import java.util.Collections; -import java.util.UUID; import org.apache.beam.sdk.io.GenerateSequence; import org.apache.beam.sdk.options.Default; @@ -54,6 +53,9 @@ /** End-to-end test of Cloud Spanner Sink. */ @RunWith(JUnit4.class) public class SpannerWriteIT { + + private static final int MAX_DB_NAME_LENGTH = 30; + @Rule public final transient TestPipeline p = TestPipeline.create(); /** Pipeline options for this test. */ @@ -113,7 +115,10 @@ public void setUp() throws Exception { } private String generateDatabaseName() { - return options.getDatabaseIdPrefix() + "-" + UUID.randomUUID().toString(); + String random = RandomStringUtils + .randomAlphanumeric(MAX_DB_NAME_LENGTH - 1 - options.getDatabaseIdPrefix().length()) + .toLowerCase(); + return options.getDatabaseIdPrefix() + "-" + random; } @Test @@ -160,7 +165,7 @@ public void processElement(ProcessContext c) { Mutation.WriteBuilder builder = Mutation.newInsertOrUpdateBuilder(table); Long key = c.element(); builder.set("Key").to(key); - builder.set("Value").to(RandomStringUtils.random(valueSize, true, true)); + builder.set("Value").to(RandomStringUtils.randomAlphabetic(valueSize)); Mutation mutation = builder.build(); c.output(mutation); }