diff --git a/benchmarks/tpcc/README.md b/benchmarks/tpcc/README.md index 2a0f7cd9d..838d16747 100644 --- a/benchmarks/tpcc/README.md +++ b/benchmarks/tpcc/README.md @@ -22,6 +22,27 @@ mvn spring-boot:run -Dspring-boot.run.arguments=" ### Load data +Load data into a PostgreSQL-dialect DB: + +```shell +mvn spring-boot:run -Dspring-boot.run.arguments=" + --tpcc.benchmark-duration=PT600s + --tpcc.warehouses=10 + --tpcc.benchmark-threads=1 + --tpcc.load-data=true + --tpcc.truncate-before-load=false + --tpcc.run-benchmark=false + --tpcc.use-read-only-transactions=false + --tpcc.lock-scanned-ranges=false + --spanner.project=my-project + --spanner.instance=my-instance + --spanner.database=my-database + --pgadapter.credentials=/path/to/credentials.json + " +``` + +To load data into a GoogleSQL-dialect DB, you need to specify `client_lib_gsql` as the `benchmark-runner`. This instructs the benchmark application that it should use the Spanner Java client to connect to the database, and that it should use the GoogleSQL dialect: + ```shell mvn spring-boot:run -Dspring-boot.run.arguments=" --tpcc.benchmark-duration=PT600s @@ -30,6 +51,7 @@ mvn spring-boot:run -Dspring-boot.run.arguments=" --tpcc.load-data=true --tpcc.truncate-before-load=false --tpcc.run-benchmark=false + --tpcc.benchmark-runner=client_lib_gsql --tpcc.use-read-only-transactions=false --tpcc.lock-scanned-ranges=false --spanner.project=my-project @@ -41,7 +63,7 @@ mvn spring-boot:run -Dspring-boot.run.arguments=" ### Run benchmark -Currently, we support benchmark runners: `pgadapter`, `spanner_jdbc`, and `client_lib_pg`. +Currently, we support the following benchmark runners: `pgadapter`, `spanner_jdbc`, `client_lib_pg`, and `client_lib_gsql`. Run with the default benchmark runner (PGAdapter with PG JDBC): @@ -102,3 +124,23 @@ mvn spring-boot:run -Dspring-boot.run.arguments=" --pgadapter.credentials=/path/to/credentials.json " ``` + +Run with the benchmark runner (Client library with GoogleSQL dialect): + +```shell +mvn spring-boot:run -Dspring-boot.run.arguments=" + --tpcc.benchmark-duration=PT600s + --tpcc.warehouses=10 + --tpcc.benchmark-threads=1 + --tpcc.load-data=false + --tpcc.truncate-before-load=false + --tpcc.run-benchmark=true + --tpcc.benchmark-runner=client_lib_gsql + --tpcc.use-read-only-transactions=true + --tpcc.lock-scanned-ranges=false + --spanner.project=my-project + --spanner.instance=my-instance + --spanner.database=my-database + --pgadapter.credentials=/path/to/credentials.json + " +``` diff --git a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/AbstractBenchmarkRunner.java b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/AbstractBenchmarkRunner.java index c37cb73da..0ec89b173 100644 --- a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/AbstractBenchmarkRunner.java +++ b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/AbstractBenchmarkRunner.java @@ -16,6 +16,7 @@ import com.google.cloud.pgadapter.tpcc.config.PGAdapterConfiguration; import com.google.cloud.pgadapter.tpcc.config.SpannerConfiguration; import com.google.cloud.pgadapter.tpcc.config.TpccConfiguration; +import com.google.cloud.spanner.Dialect; import com.google.cloud.spanner.jdbc.JdbcSqlExceptionFactory.JdbcAbortedException; import java.io.IOException; import java.math.BigDecimal; @@ -44,6 +45,7 @@ abstract class AbstractBenchmarkRunner implements Runnable { protected final PGAdapterConfiguration pgAdapterConfiguration; protected final SpannerConfiguration spannerConfiguration; protected final Metrics metrics; + protected final Dialect dialect; private boolean failed; @@ -52,12 +54,14 @@ abstract class AbstractBenchmarkRunner implements Runnable { TpccConfiguration tpccConfiguration, PGAdapterConfiguration pgAdapterConfiguration, SpannerConfiguration spannerConfiguration, - Metrics metrics) { + Metrics metrics, + Dialect dialect) { this.statistics = statistics; this.tpccConfiguration = tpccConfiguration; this.pgAdapterConfiguration = pgAdapterConfiguration; this.spannerConfiguration = spannerConfiguration; this.metrics = metrics; + this.dialect = dialect; } @Override @@ -201,7 +205,7 @@ private void newOrder() throws SQLException { new Object[] {districtNextOrderId + 1L, districtId, warehouseId}); executeParamStatement( "INSERT INTO orders (o_id, d_id, w_id, c_id, o_entry_d, o_ol_cnt, o_all_local) " - + "VALUES (?,?,?,?,NOW(),?,?)", + + "VALUES (?,?,?,?,CURRENT_TIMESTAMP,?,?)", new Object[] { districtNextOrderId, districtId, warehouseId, customerId, orderLineCount, allLocal }); @@ -436,7 +440,7 @@ private void payment() throws SQLException { } executeParamStatement( "INSERT INTO history (d_id, w_id, c_id, h_d_id, h_w_id, h_date, h_amount, h_data) " - + "VALUES (?,?,?,?,?,NOW(),?,?)", + + "VALUES (?,?,?,?,?,CURRENT_TIMESTAMP,?,?)", new Object[] { customerDistrictId, customerWarehouseId, @@ -580,7 +584,7 @@ private void delivery() throws SQLException { new Object[] {carrierId, newOrderId, customerId, districtId, warehouseId}); executeParamStatement( "UPDATE order_line " - + "SET ol_delivery_d = NOW() " + + "SET ol_delivery_d = CURRENT_TIMESTAMP " + "WHERE o_id = ? AND c_id = ? AND d_id = ? AND w_id = ?", new Object[] {newOrderId, customerId, districtId, warehouseId}); row = diff --git a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/BenchmarkApplication.java b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/BenchmarkApplication.java index 7483e5d2c..1f2f704e5 100644 --- a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/BenchmarkApplication.java +++ b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/BenchmarkApplication.java @@ -20,6 +20,7 @@ import com.google.cloud.pgadapter.tpcc.config.TpccConfiguration; import com.google.cloud.pgadapter.tpcc.dataloader.DataLoadStatus; import com.google.cloud.pgadapter.tpcc.dataloader.DataLoader; +import com.google.cloud.spanner.Dialect; import com.google.cloud.spanner.SessionPoolOptions; import com.google.cloud.spanner.SpannerOptions; import com.google.cloud.spanner.pgadapter.ProxyServer; @@ -97,8 +98,13 @@ public void run(String... args) throws Exception { pgAdapterConfiguration.getMaxSessions(), tpccConfiguration.getBenchmarkThreads())); try { if (tpccConfiguration.isLoadData()) { + boolean isClientLibGSQLRunner = + tpccConfiguration.getBenchmarkRunner().equals(TpccConfiguration.CLIENT_LIB_GSQL_RUNNER); + Dialect dialect = isClientLibGSQLRunner ? Dialect.GOOGLE_STANDARD_SQL : Dialect.POSTGRESQL; + String loadDataConnectionUrl = + isClientLibGSQLRunner ? spannerConnectionUrl : pgadapterConnectionUrl; System.out.println("Checking schema"); - SchemaService schemaService = new SchemaService(pgadapterConnectionUrl); + SchemaService schemaService = new SchemaService(loadDataConnectionUrl, dialect); schemaService.createSchema(); System.out.println("Checked schema, starting benchmark"); @@ -106,7 +112,7 @@ public void run(String... args) throws Exception { ExecutorService executor = Executors.newSingleThreadExecutor(); DataLoadStatus status = new DataLoadStatus(tpccConfiguration); Future loadDataFuture = - executor.submit(() -> loadData(status, pgadapterConnectionUrl)); + executor.submit(() -> loadData(status, loadDataConnectionUrl)); executor.shutdown(); Stopwatch watch = Stopwatch.createStarted(); while (!loadDataFuture.isDone()) { @@ -166,7 +172,21 @@ public void run(String... args) throws Exception { tpccConfiguration, pgAdapterConfiguration, spannerConfiguration, - metrics)); + metrics, + Dialect.POSTGRESQL)); + } else if (tpccConfiguration + .getBenchmarkRunner() + .equals(TpccConfiguration.CLIENT_LIB_GSQL_RUNNER)) { + // Run client library PG benchmark + statistics.setRunnerName("Client library GSQL benchmark"); + executor.submit( + new JavaClientBenchmarkRunner( + statistics, + tpccConfiguration, + pgAdapterConfiguration, + spannerConfiguration, + metrics, + Dialect.GOOGLE_STANDARD_SQL)); } } diff --git a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/JavaClientBenchmarkRunner.java b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/JavaClientBenchmarkRunner.java index c5de2731e..69744c9a1 100644 --- a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/JavaClientBenchmarkRunner.java +++ b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/JavaClientBenchmarkRunner.java @@ -17,9 +17,9 @@ import com.google.cloud.pgadapter.tpcc.config.PGAdapterConfiguration; import com.google.cloud.pgadapter.tpcc.config.SpannerConfiguration; import com.google.cloud.pgadapter.tpcc.config.TpccConfiguration; -import com.google.cloud.spanner.AbortedException; import com.google.cloud.spanner.DatabaseClient; import com.google.cloud.spanner.DatabaseId; +import com.google.cloud.spanner.Dialect; import com.google.cloud.spanner.ResultSet; import com.google.cloud.spanner.SessionPoolOptions; import com.google.cloud.spanner.Spanner; @@ -42,6 +42,7 @@ import org.slf4j.LoggerFactory; class JavaClientBenchmarkRunner extends AbstractBenchmarkRunner { + private static final Logger LOG = LoggerFactory.getLogger(JavaClientBenchmarkRunner.class); private Spanner spanner; private final ThreadLocal transactionManagerThreadLocal = new ThreadLocal<>(); @@ -52,8 +53,15 @@ class JavaClientBenchmarkRunner extends AbstractBenchmarkRunner { TpccConfiguration tpccConfiguration, PGAdapterConfiguration pgAdapterConfiguration, SpannerConfiguration spannerConfiguration, - Metrics metrics) { - super(statistics, tpccConfiguration, pgAdapterConfiguration, spannerConfiguration, metrics); + Metrics metrics, + Dialect dialect) { + super( + statistics, + tpccConfiguration, + pgAdapterConfiguration, + spannerConfiguration, + metrics, + dialect); } void setup() throws SQLException, IOException { @@ -80,9 +88,6 @@ void executeStatement(String sql) throws SQLException { transactionManagerThreadLocal.get().commit(); Duration executionDuration = stopwatch.elapsed(); metrics.recordLatency(executionDuration.toMillis()); - } catch (AbortedException e) { - // Ignore the aborted exception. Roll back the transaction. - transactionManagerThreadLocal.get().rollback(); } finally { transactionManagerThreadLocal.get().close(); } @@ -98,7 +103,11 @@ void bindParams(Statement.Builder builder, Object[] params) throws SQLException for (int i = 0; i < params.length; i++) { Object param = params[i]; if (param instanceof BigDecimal) { - builder.bind("p" + (i + 1)).to(Value.pgNumeric(((BigDecimal) param).toPlainString())); + if (dialect == Dialect.POSTGRESQL) { + builder.bind("p" + (i + 1)).to(Value.pgNumeric(((BigDecimal) param).toPlainString())); + } else { + builder.bind("p" + (i + 1)).to((BigDecimal) param); + } } else if (param instanceof String) { builder.bind("p" + (i + 1)).to((String) param); } else if (param instanceof Integer) { @@ -112,7 +121,8 @@ void bindParams(Statement.Builder builder, Object[] params) throws SQLException } Object[] paramQueryRow(String sql, Object[] params) throws SQLException { - ParametersInfo parametersInfo = convertPositionalParametersToNamedParameters(sql, '?', "$"); + ParametersInfo parametersInfo = + convertPositionalParametersToNamedParameters(sql, '?', getNamedParameterPrefix()); Statement.Builder builder = Statement.newBuilder(parametersInfo.sqlWithNamedParameters); bindParams(builder, params); Statement stmt = builder.build(); @@ -137,7 +147,8 @@ Object[] paramQueryRow(String sql, Object[] params) throws SQLException { } void executeParamStatement(String sql, Object[] params) throws SQLException { - ParametersInfo parametersInfo = convertPositionalParametersToNamedParameters(sql, '?', "$"); + ParametersInfo parametersInfo = + convertPositionalParametersToNamedParameters(sql, '?', getNamedParameterPrefix()); Statement.Builder builder = Statement.newBuilder(parametersInfo.sqlWithNamedParameters); bindParams(builder, params); Statement stmt = builder.build(); @@ -196,7 +207,9 @@ static int countOccurrencesOf(char c, String string) { // If getAsObject() in Struct can be a public method, then we should use it // instead of using the following approach. Object getObject(Value value) { - if (value.getType().equals(Type.numeric())) { + if (value == null || value.isNull()) { + return null; + } else if (value.getType().equals(Type.numeric())) { return value.getNumeric(); } else if (value.getType().equals(Type.string())) { return value.getString(); @@ -212,8 +225,13 @@ Object getObject(Value value) { return null; } + String getNamedParameterPrefix() { + return dialect == Dialect.POSTGRESQL ? "$" : "@p"; + } + List executeParamQuery(String sql, Object[] params) throws SQLException { - ParametersInfo parametersInfo = convertPositionalParametersToNamedParameters(sql, '?', "$"); + ParametersInfo parametersInfo = + convertPositionalParametersToNamedParameters(sql, '?', getNamedParameterPrefix()); List results = new ArrayList<>(); Statement.Builder builder = Statement.newBuilder(parametersInfo.sqlWithNamedParameters); diff --git a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/JdbcBenchmarkRunner.java b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/JdbcBenchmarkRunner.java index 8c973b071..8c86fb745 100644 --- a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/JdbcBenchmarkRunner.java +++ b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/JdbcBenchmarkRunner.java @@ -16,6 +16,7 @@ import com.google.cloud.pgadapter.tpcc.config.PGAdapterConfiguration; import com.google.cloud.pgadapter.tpcc.config.SpannerConfiguration; import com.google.cloud.pgadapter.tpcc.config.TpccConfiguration; +import com.google.cloud.spanner.Dialect; import com.google.common.base.Stopwatch; import java.io.IOException; import java.math.BigDecimal; @@ -46,7 +47,13 @@ class JdbcBenchmarkRunner extends AbstractBenchmarkRunner { PGAdapterConfiguration pgAdapterConfiguration, SpannerConfiguration spannerConfiguration, Metrics metrics) { - super(statistics, tpccConfiguration, pgAdapterConfiguration, spannerConfiguration, metrics); + super( + statistics, + tpccConfiguration, + pgAdapterConfiguration, + spannerConfiguration, + metrics, + Dialect.POSTGRESQL); this.connectionUrl = connectionUrl; } diff --git a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/SchemaService.java b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/SchemaService.java index 43befabc3..8d2b599e2 100644 --- a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/SchemaService.java +++ b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/SchemaService.java @@ -13,6 +13,7 @@ // limitations under the License. package com.google.cloud.pgadapter.tpcc; +import com.google.cloud.spanner.Dialect; import java.io.IOException; import java.net.URL; import java.nio.file.Files; @@ -31,11 +32,16 @@ class SchemaService { private final String connectionUrl; - SchemaService(String connectionUrl) { + private final Dialect dialect; + + SchemaService(String connectionUrl, Dialect dialect) { this.connectionUrl = connectionUrl; + this.dialect = dialect; } void createSchema() throws IOException, SQLException { + LOG.info("Schema connection URL: " + connectionUrl); + boolean isGoogleSQL = dialect == Dialect.GOOGLE_STANDARD_SQL; try (Connection connection = DriverManager.getConnection(connectionUrl)) { // Check if the tables already exist. try (ResultSet resultSet = @@ -44,21 +50,24 @@ void createSchema() throws IOException, SQLException { .executeQuery( "select count(1) " + "from information_schema.tables " - + "where table_schema='public' " - + "and table_name in ('warehouse', 'district', 'customer', 'history', 'orders', 'new_orders', 'order_line', 'stock', 'item')")) { + + "where " + + (isGoogleSQL ? "table_schema='' and " : "table_schema='public' and ") + + "table_name in ('warehouse', 'district', 'customer', 'history', 'orders', 'new_orders', 'order_line', 'stock', 'item')")) { if (resultSet.next() && resultSet.getInt(1) == 9) { LOG.info("Skipping schema creation as tables already exist"); return; } } - URL url = AbstractBenchmarkRunner.class.getResource("/schema.sql"); + URL url = + AbstractBenchmarkRunner.class.getResource( + isGoogleSQL ? "/schema_googlesql.sql" : "/schema.sql"); Path path = Paths.get(Objects.requireNonNull(url).getPath()); String ddl = Files.readString(path); LOG.info("Executing schema statements"); - String[] statements = ddl.split(";"); + String[] statements = ddl.trim().split(";"); for (String statement : statements) { - LOG.info(statement); + LOG.info("Statement: " + statement); connection.createStatement().execute(statement); } } diff --git a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/config/TpccConfiguration.java b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/config/TpccConfiguration.java index 8ee22457f..af0ac14ad 100644 --- a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/config/TpccConfiguration.java +++ b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/config/TpccConfiguration.java @@ -24,8 +24,10 @@ public class TpccConfiguration { public static final String PGADAPTER_JDBC_RUNNER = "pgadapter"; public static final String SPANNER_JDBC_RUNNER = "spanner_jdbc"; public static final String CLIENT_LIB_PG_RUNNER = "client_lib_pg"; + public static final String CLIENT_LIB_GSQL_RUNNER = "client_lib_gsql"; public static final Set RUNNERS = - Set.of(PGADAPTER_JDBC_RUNNER, SPANNER_JDBC_RUNNER, CLIENT_LIB_PG_RUNNER); + Set.of( + PGADAPTER_JDBC_RUNNER, SPANNER_JDBC_RUNNER, CLIENT_LIB_PG_RUNNER, CLIENT_LIB_GSQL_RUNNER); private boolean loadData; diff --git a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/AbstractRowProducer.java b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/AbstractRowProducer.java index bb28e77ae..ef5c28e82 100644 --- a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/AbstractRowProducer.java +++ b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/AbstractRowProducer.java @@ -13,6 +13,7 @@ // limitations under the License. package com.google.cloud.pgadapter.tpcc.dataloader; +import com.google.common.collect.ImmutableList; import java.io.Writer; import java.math.BigDecimal; import java.math.MathContext; @@ -21,6 +22,8 @@ import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import java.util.Arrays; +import java.util.List; import java.util.Random; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; @@ -29,17 +32,29 @@ abstract class AbstractRowProducer { private final String table; private final String columns; + private final List columnList; private final long rowCount; private final Runnable rowCounterIncrementer; + protected Long warehouseId = null; + protected Long districtId = null; final Random random = new Random(); AbstractRowProducer(String table, String columns, long rowCount, Runnable rowCounterIncrementer) { this.table = table; this.columns = columns; + String[] parts = columns.split(","); + for (int i = 0; i < parts.length; i++) { + parts[i] = parts[i].trim().replace("\n", ""); + } + this.columnList = Arrays.asList(parts); this.rowCount = rowCount; this.rowCounterIncrementer = rowCounterIncrementer; } + void incRowCounterIncrementer() { + rowCounterIncrementer.run(); + } + String getTable() { return table; } @@ -48,6 +63,14 @@ String getColumns() { return columns; } + List getColumnsAsList() { + return columnList; + } + + long getRowCount() { + return rowCount; + } + Future asyncWriteRows(ExecutorService executor, Writer writer) { return executor.submit( () -> { @@ -64,7 +87,22 @@ Future asyncWriteRows(ExecutorService executor, Writer writer) { }); } - abstract String createRow(long rowIndex); + String createRow(long rowIndex) { + List list = createRowsAsList(rowIndex); + if (list == null) { + return null; + } + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < list.size(); i++) { + if (i > 0) { + builder.append("\n"); + } + builder.append(String.join(",", list.get(i))); + } + return builder.toString(); + } + + abstract List createRowsAsList(long rowIndex); String quote(String input) { return "'" + input + "'"; @@ -95,7 +133,7 @@ String getRandomZip() { } String getRandomTax() { - return getRandomDecimal(2); + return getRandomDecimal(2).toPlainString(); } String getRandomString(int length) { @@ -111,21 +149,20 @@ String getRandomInt(int min, int max) { return String.valueOf(random.nextInt(max - min + 1) + min); } - String getRandomDecimal(int precision) { + BigDecimal getRandomDecimal(int precision) { return getRandomDecimal(1, precision); } - String getRandomDecimal(int factor, int precision) { + BigDecimal getRandomDecimal(int factor, int precision) { return BigDecimal.valueOf(random.nextDouble() * factor) - .round(new MathContext(precision, RoundingMode.HALF_UP)) - .toPlainString(); + .round(new MathContext(precision, RoundingMode.HALF_UP)); } String getRandomPhone() { return getRandomInt(100_000_000, 999_999_999); } - String now() { + String nowAsString() { return DateTimeFormatter.ISO_OFFSET_DATE_TIME .format(ZonedDateTime.ofInstant(Instant.now(), ZoneId.systemDefault())) .replace('T', ' '); diff --git a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/CustomerRowProducer.java b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/CustomerRowProducer.java index fbb63fe54..12cbede22 100644 --- a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/CustomerRowProducer.java +++ b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/CustomerRowProducer.java @@ -13,8 +13,12 @@ // limitations under the License. package com.google.cloud.pgadapter.tpcc.dataloader; +import com.google.cloud.Timestamp; import com.google.cloud.pgadapter.tpcc.LastNameGenerator; import com.google.common.collect.ImmutableList; +import java.math.BigDecimal; +import java.util.Arrays; +import java.util.List; class CustomerRowProducer extends AbstractRowProducer { private static final String TABLE = "customer"; @@ -43,10 +47,6 @@ class CustomerRowProducer extends AbstractRowProducer { c_data """; - private final long warehouseId; - - private final long districtId; - CustomerRowProducer(DataLoadStatus status, long warehouseId, long districtId, long rowCount) { super(TABLE, COLUMNS, rowCount, status::incCustomer); this.warehouseId = warehouseId; @@ -70,7 +70,34 @@ String createRow(long rowIndex) { getRandomState(), getRandomZip(), getRandomPhone(), - now(), + nowAsString(), // Use a timestamp as string. + getCredit(), + getCreditLimit(), + getDiscount().toPlainString(), + getBalance(), + getYtdPayment(), + getPaymentCount(), + getDeliveryCount(), + getData())); + } + + @Override + List createRowsAsList(long rowIndex) { + return Arrays.asList( + ImmutableList.of( + getId(rowIndex), + String.valueOf(districtId), + String.valueOf(warehouseId), + getRandomName(), + getRandomString(2), + getLastName(rowIndex), + getRandomStreet(1), + getRandomStreet(2), + getRandomCity(), + getRandomState(), + getRandomZip(), + getRandomPhone(), + Timestamp.now(), getCredit(), getCreditLimit(), getDiscount(), @@ -93,7 +120,7 @@ String getCreditLimit() { return "50000"; } - String getDiscount() { + BigDecimal getDiscount() { return getRandomDecimal(2); } diff --git a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/DataLoader.java b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/DataLoader.java index 3938437b8..97a4d07f4 100644 --- a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/DataLoader.java +++ b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/DataLoader.java @@ -13,8 +13,15 @@ // limitations under the License. package com.google.cloud.pgadapter.tpcc.dataloader; +import com.google.cloud.Timestamp; import com.google.cloud.pgadapter.tpcc.BenchmarkApplication; import com.google.cloud.pgadapter.tpcc.config.TpccConfiguration; +import com.google.cloud.spanner.Dialect; +import com.google.cloud.spanner.Mutation; +import com.google.cloud.spanner.Mutation.WriteBuilder; +import com.google.cloud.spanner.ValueBinder; +import com.google.cloud.spanner.jdbc.CloudSpannerJdbcConnection; +import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; @@ -22,6 +29,7 @@ import java.io.IOException; import java.io.PipedReader; import java.io.PipedWriter; +import java.math.BigDecimal; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; @@ -39,6 +47,8 @@ public class DataLoader implements AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(BenchmarkApplication.class); + private final int MUTATION_LIMIT_PER_COMMIT = 60000; + private final String connectionUrl; private final TpccConfiguration tpccConfiguration; @@ -49,6 +59,8 @@ public class DataLoader implements AutoCloseable { private final DataLoadStatus status; + private final Dialect dialect; + public DataLoader( DataLoadStatus status, String connectionUrl, TpccConfiguration tpccConfiguration) { this.connectionUrl = connectionUrl; @@ -60,6 +72,10 @@ public DataLoader( MoreExecutors.listeningDecorator( Executors.newFixedThreadPool(tpccConfiguration.getLoadDataThreads())); this.status = status; + this.dialect = + tpccConfiguration.getBenchmarkRunner().equals(TpccConfiguration.CLIENT_LIB_GSQL_RUNNER) + ? Dialect.GOOGLE_STANDARD_SQL + : Dialect.POSTGRESQL; } @Override @@ -270,7 +286,128 @@ public long loadData() throws Exception { return totalRowCount; } - long loadTable(AbstractRowProducer rowProducer) throws SQLException, IOException { + long loadTable(AbstractRowProducer rowProducer) + throws RuntimeException, SQLException, IOException { + if (dialect == Dialect.GOOGLE_STANDARD_SQL) { + return loadTableInGSQL(rowProducer); + } + return loadTableInPG(rowProducer); + } + + String unquote(String input) { + if (input.length() >= 2 && input.startsWith("'") && input.endsWith("'")) { + return input.substring(1, input.length() - 1); + } + return input; + } + + void setMutationColumn(WriteBuilder writeBuilder, String name, Object value) + throws RuntimeException { + ValueBinder valueBinder = writeBuilder.set(name); + if (value instanceof Boolean) { + valueBinder.to((Boolean) value); + } else if (value instanceof Long) { + valueBinder.to((Long) value); + } else if (value instanceof Float) { + valueBinder.to((Float) value); + } else if (value instanceof Double) { + valueBinder.to((Double) value); + } else if (value instanceof BigDecimal) { + valueBinder.to((BigDecimal) value); + } else if (value instanceof Timestamp) { + Timestamp timestamp = (Timestamp) value; + if (timestamp.equals(Timestamp.MIN_VALUE)) { + // We use Timestamp.MIN_VALUE to represent NULL as ImmutableList does not permit + // NULLs. + valueBinder.to((Timestamp) null); + } else { + valueBinder.to(timestamp); + } + } else if (value instanceof String) { + // Unquote the string because the generated strings are wrapped with single + // quotes. + String stringValue = unquote((String) value); + if (stringValue.isEmpty()) { + valueBinder.to((String) null); + } else { + valueBinder.to(stringValue); + } + } else { + throw new RuntimeException(String.format("Unknown value for column %s: %s", name, value)); + } + } + + long loadTableInGSQL(AbstractRowProducer rowProducer) + throws RuntimeException, SQLException, IOException { + long count = 0; + try (Connection connection = createConnection(); + Statement statement = connection.createStatement()) { + // Unwrap the Cloud Spanner specific interface to be able to access custom + // methods. + CloudSpannerJdbcConnection spannerConnection = + connection.unwrap(CloudSpannerJdbcConnection.class); + spannerConnection.setAutoCommit(false); + + // The smallest table `new_orders` only has 4 columns. So the largest number of + // mutations that we may need to store in a batch can be the predefined mutation + // limit divided by 4. + List mutations = new ArrayList<>(Math.round(MUTATION_LIMIT_PER_COMMIT / 4)); + int columnCount = rowProducer.getColumnsAsList().size(); + for (long rowIndex = 0L; rowIndex < rowProducer.getRowCount(); rowIndex++) { + List rows = rowProducer.createRowsAsList(rowIndex); + if (rows == null) { + continue; + } + for (int i = 0; i < rows.size(); i++) { + ImmutableList row = rows.get(i); + WriteBuilder writeBuilder = Mutation.newInsertBuilder(rowProducer.getTable()); + if (columnCount != row.size()) { + // This should never happen. + throw new RuntimeException( + String.format( + "The number of generated columns does not match the target number - expected: %d, actual: %d", + columnCount, row.size())); + } + for (int columnIndex = 0; columnIndex < columnCount; columnIndex++) { + setMutationColumn( + writeBuilder, + rowProducer.getColumnsAsList().get(columnIndex), + row.get(columnIndex)); + } + Mutation mutation = writeBuilder.build(); + mutations.add(mutation); + } + + if ((mutations.size() * columnCount) >= MUTATION_LIMIT_PER_COMMIT) { + spannerConnection.bufferedWrite(mutations); + spannerConnection.commit(); + count += mutations.size(); + mutations.clear(); + } + + // We need to put this increase here because `rows` only has multiple records + // for `order_line` table. The number of rows in the `order_line` table is + // greater than the computed total number: + // count(`order_line` table) = warehouses * districts + // * customers per district + // * a pseudo-random value + // When we show the rows in the status, we only a number without the + // pseudo-random value. + rowProducer.incRowCounterIncrementer(); + } + + // This is needed because there are some uncommitted mutations. + if (mutations.size() != 0) { + spannerConnection.bufferedWrite(mutations); + spannerConnection.commit(); + count += mutations.size(); + } + + return count; + } + } + + long loadTableInPG(AbstractRowProducer rowProducer) throws SQLException, IOException { PipedWriter writer = new PipedWriter(); try (PipedReader reader = new PipedReader(writer, 30_000); Connection connection = createConnection()) { @@ -284,48 +421,66 @@ long loadTable(AbstractRowProducer rowProducer) throws SQLException, IOException } } + void truncateTable(Statement statement, String table) throws SQLException { + if (dialect == Dialect.GOOGLE_STANDARD_SQL) { + statement.execute(String.format("delete from %s where true", table)); + } else { + statement.execute("truncate table " + table); + } + } + void truncate() throws SQLException { try (Connection connection = createConnection(); Statement statement = connection.createStatement()) { + if (dialect == Dialect.GOOGLE_STANDARD_SQL) { + statement.execute("set autocommit=true"); + } LOG.info("truncating new_orders"); - statement.execute("truncate table new_orders"); + truncateTable(statement, "new_orders"); status.setTruncatedNewOrders(); LOG.info("truncating order_line"); - statement.execute("truncate table order_line"); + truncateTable(statement, "order_line"); status.setTruncatedOrderLine(); LOG.info("truncating orders"); - statement.execute("truncate table orders"); + truncateTable(statement, "orders"); status.setTruncatedOrders(); LOG.info("truncating history"); - statement.execute("truncate table history"); + truncateTable(statement, "history"); status.setTruncatedHistory(); LOG.info("truncating customer"); - statement.execute("truncate table customer"); + truncateTable(statement, "customer"); status.setTruncatedCustomer(); LOG.info("truncating stock"); - statement.execute("truncate table stock"); + truncateTable(statement, "stock"); status.setTruncatedStock(); LOG.info("truncating district"); - statement.execute("truncate table district"); + truncateTable(statement, "district"); status.setTruncatedDistrict(); LOG.info("truncating warehouse"); - statement.execute("truncate table warehouse"); + truncateTable(statement, "warehouse"); status.setTruncatedWarehouse(); LOG.info("truncating item"); - statement.execute("truncate table item"); + truncateTable(statement, "item"); status.setTruncatedItem(); + if (dialect == Dialect.GOOGLE_STANDARD_SQL) { + statement.execute("set autocommit=false"); + } } } private Connection createConnection() throws SQLException { Connection connection = DriverManager.getConnection(connectionUrl); - // Use upsert instead of insert for COPY to prevent data loading errors if the tables are - // already half-filled. - connection.createStatement().execute("set spanner.copy_upsert=true"); // Allow copy operations to be non-atomic. - connection - .createStatement() - .execute("set spanner.autocommit_dml_mode='partitioned_non_atomic'"); + if (dialect == Dialect.GOOGLE_STANDARD_SQL) { + connection.createStatement().execute("set autocommit_dml_mode='partitioned_non_atomic'"); + } else { + // Use upsert instead of insert for COPY to prevent data loading errors if the + // tables are already half-filled. + connection.createStatement().execute("set spanner.copy_upsert=true"); + connection + .createStatement() + .execute("set spanner.autocommit_dml_mode='partitioned_non_atomic'"); + } return connection; } } diff --git a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/DistrictRowProducer.java b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/DistrictRowProducer.java index 896afffc5..9995c4643 100644 --- a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/DistrictRowProducer.java +++ b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/DistrictRowProducer.java @@ -14,6 +14,8 @@ package com.google.cloud.pgadapter.tpcc.dataloader; import com.google.common.collect.ImmutableList; +import java.util.Arrays; +import java.util.List; class DistrictRowProducer extends AbstractRowProducer { private static final String TABLE = "district"; @@ -31,17 +33,14 @@ class DistrictRowProducer extends AbstractRowProducer { d_ytd, d_next_o_id"""; - private final long warehouseId; - DistrictRowProducer(DataLoadStatus status, long warehouseId, long rowCount) { super(TABLE, COLUMNS, rowCount, status::incDistrict); this.warehouseId = warehouseId; } @Override - String createRow(long rowIndex) { - return String.join( - ",", + List createRowsAsList(long rowIndex) { + return Arrays.asList( ImmutableList.of( getId(rowIndex), String.valueOf(warehouseId), diff --git a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/HistoryRowProducer.java b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/HistoryRowProducer.java index 6179d03fc..f0de55424 100644 --- a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/HistoryRowProducer.java +++ b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/HistoryRowProducer.java @@ -13,7 +13,10 @@ // limitations under the License. package com.google.cloud.pgadapter.tpcc.dataloader; +import com.google.cloud.Timestamp; import com.google.common.collect.ImmutableList; +import java.util.Arrays; +import java.util.List; class HistoryRowProducer extends AbstractRowProducer { private static final String TABLE = "history"; @@ -29,10 +32,6 @@ class HistoryRowProducer extends AbstractRowProducer { h_data """; - private final long warehouseId; - - private final long districtId; - HistoryRowProducer(DataLoadStatus status, long warehouseId, long districtId, long rowCount) { super(TABLE, COLUMNS, rowCount, status::incHistory); this.warehouseId = warehouseId; @@ -49,7 +48,21 @@ String createRow(long rowIndex) { String.valueOf(warehouseId), String.valueOf(districtId), String.valueOf(warehouseId), - now(), + nowAsString(), + getYtdPayment(), + getData())); + } + + @Override + List createRowsAsList(long rowIndex) { + return Arrays.asList( + ImmutableList.of( + getId(rowIndex), + String.valueOf(districtId), + String.valueOf(warehouseId), + String.valueOf(districtId), + String.valueOf(warehouseId), + Timestamp.now(), getYtdPayment(), getData())); } diff --git a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/ItemRowProducer.java b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/ItemRowProducer.java index 344024721..e3dde0dc6 100644 --- a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/ItemRowProducer.java +++ b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/ItemRowProducer.java @@ -14,6 +14,9 @@ package com.google.cloud.pgadapter.tpcc.dataloader; import com.google.common.collect.ImmutableList; +import java.math.BigDecimal; +import java.util.Arrays; +import java.util.List; class ItemRowProducer extends AbstractRowProducer { private static final String TABLE = "item"; @@ -34,6 +37,17 @@ class ItemRowProducer extends AbstractRowProducer { String createRow(long rowIndex) { return String.join( ",", + ImmutableList.of( + getId(rowIndex), + getRandomImId(), + getName(rowIndex), + getRandomPrice().toPlainString(), + getData())); + } + + @Override + List createRowsAsList(long rowIndex) { + return Arrays.asList( ImmutableList.of( getId(rowIndex), getRandomImId(), getName(rowIndex), getRandomPrice(), getData())); } @@ -42,7 +56,7 @@ String getRandomImId() { return getRandomInt(1, 10000); } - String getRandomPrice() { + BigDecimal getRandomPrice() { return getRandomDecimal(100, 2); } diff --git a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/NewOrderRowProducer.java b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/NewOrderRowProducer.java index 1680c02ac..6ba925b25 100644 --- a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/NewOrderRowProducer.java +++ b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/NewOrderRowProducer.java @@ -15,6 +15,8 @@ import com.google.cloud.pgadapter.tpcc.dataloader.OrderRowProducer.DistrictId; import com.google.common.collect.ImmutableList; +import java.util.Arrays; +import java.util.List; class NewOrderRowProducer extends AbstractOrderedIdRowProducer { private static final String TABLE = "new_orders"; @@ -26,10 +28,6 @@ class NewOrderRowProducer extends AbstractOrderedIdRowProducer { w_id """; - private final long warehouseId; - - private final long districtId; - private final ImmutableList customerIds; NewOrderRowProducer(DataLoadStatus status, long warehouseId, long districtId, long rowCount) { @@ -40,10 +38,9 @@ class NewOrderRowProducer extends AbstractOrderedIdRowProducer { } @Override - String createRow(long rowIndex) { + List createRowsAsList(long rowIndex) { if (rowIndex % 3L == 0) { - return String.join( - ",", + return Arrays.asList( ImmutableList.of( getId(rowIndex), getCustomerId(rowIndex), diff --git a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/OrderLineRowProducer.java b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/OrderLineRowProducer.java index e2b1990d9..d0acdd4fc 100644 --- a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/OrderLineRowProducer.java +++ b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/OrderLineRowProducer.java @@ -13,8 +13,12 @@ // limitations under the License. package com.google.cloud.pgadapter.tpcc.dataloader; +import com.google.cloud.Timestamp; import com.google.cloud.pgadapter.tpcc.dataloader.OrderRowProducer.DistrictId; import com.google.common.collect.ImmutableList; +import java.math.BigDecimal; +import java.util.Arrays; +import java.util.List; import java.util.Objects; class OrderLineRowProducer extends AbstractOrderedIdRowProducer { @@ -34,10 +38,6 @@ class OrderLineRowProducer extends AbstractOrderedIdRowProducer { ol_dist_info """; - private final long warehouseId; - - private final long districtId; - private final int itemCount; private final ImmutableList customerIds; @@ -69,14 +69,35 @@ String createRow(long rowIndex) { String.valueOf(line), getRandomItem(), String.valueOf(warehouseId), - getDeliveryDate(rowIndex), + getDeliveryDateAsString(rowIndex), getQuantity(), - getPrice(rowIndex), + getPrice(rowIndex).toPlainString(), getData()))); } return builder.toString(); } + @Override + List createRowsAsList(long rowIndex) { + ImmutableList[] array = new ImmutableList[getOrderLineCount(rowIndex)]; + for (int line = 0; line < getOrderLineCount(rowIndex); line++) { + array[line] = + ImmutableList.of( + getId(rowIndex), + getCustomerId(rowIndex), + String.valueOf(districtId), + String.valueOf(warehouseId), + String.valueOf(line), + getRandomItem(), + String.valueOf(warehouseId), + getDeliveryDate(rowIndex), + getQuantity(), + getPrice(rowIndex), + getData()); + } + return Arrays.asList(array); + } + String getCustomerId(long rowIndex) { return String.valueOf(Long.reverse(customerIds.get((int) rowIndex))); } @@ -89,16 +110,22 @@ String getRandomItem() { return String.valueOf(Long.reverse(random.nextInt(itemCount))); } - String getDeliveryDate(long rowIndex) { - return rowIndex % 3L == 0 ? "" : now(); + Timestamp getDeliveryDate(long rowIndex) { + // Use smallest timestamp to represent NULL because ImmutableList does not + // permit NULLs. + return rowIndex % 3L == 0 ? Timestamp.MIN_VALUE : Timestamp.now(); + } + + String getDeliveryDateAsString(long rowIndex) { + return rowIndex % 3L == 0 ? "" : nowAsString(); } String getQuantity() { return "5"; } - String getPrice(long rowIndex) { - return rowIndex % 3L == 0 ? "0.0" : getRandomDecimal(10000, 2); + BigDecimal getPrice(long rowIndex) { + return rowIndex % 3L == 0 ? BigDecimal.valueOf(0.0) : getRandomDecimal(10000, 2); } String getData() { diff --git a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/OrderRowProducer.java b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/OrderRowProducer.java index e411c8bc9..673e73cdf 100644 --- a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/OrderRowProducer.java +++ b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/OrderRowProducer.java @@ -13,16 +13,22 @@ // limitations under the License. package com.google.cloud.pgadapter.tpcc.dataloader; +import com.google.cloud.Timestamp; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.Objects; +import java.util.Random; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.stream.LongStream; class OrderRowProducer extends AbstractOrderedIdRowProducer { + private static final long SEED = 12345; + static class DistrictId { final long warehouse; final long district; @@ -62,10 +68,6 @@ public boolean equals(Object o) { o_all_local """; - private final long warehouseId; - - private final long districtId; - private final ImmutableList customerIds; OrderRowProducer( @@ -75,7 +77,9 @@ public boolean equals(Object o) { this.districtId = districtId; ArrayList customerIds = Lists.newArrayList(LongStream.range(0L, customerCount).iterator()); - Collections.shuffle(customerIds); + // We need this to be deterministic; otherwise, the new_orders table cannot find + // the right reference and break the foreign key. + Collections.shuffle(customerIds, new Random(SEED)); this.customerIds = ImmutableList.copyOf(customerIds); CUSTOMER_IDS.put(new DistrictId(warehouseId, districtId), this.customerIds); } @@ -89,7 +93,21 @@ String createRow(long rowIndex) { String.valueOf(districtId), String.valueOf(warehouseId), getCustomerId(rowIndex), - now(), + nowAsString(), + getRandomCarrierId(rowIndex), + getOrderLineCount(rowIndex), + getAllLocal())); + } + + @Override + List createRowsAsList(long rowIndex) { + return Arrays.asList( + ImmutableList.of( + getId(rowIndex), + String.valueOf(districtId), + String.valueOf(warehouseId), + getCustomerId(rowIndex), + Timestamp.now(), getRandomCarrierId(rowIndex), getOrderLineCount(rowIndex), getAllLocal())); diff --git a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/StockRowProducer.java b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/StockRowProducer.java index 7550f284b..569a2d15f 100644 --- a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/StockRowProducer.java +++ b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/StockRowProducer.java @@ -14,6 +14,8 @@ package com.google.cloud.pgadapter.tpcc.dataloader; import com.google.common.collect.ImmutableList; +import java.util.Arrays; +import java.util.List; class StockRowProducer extends AbstractRowProducer { private static final String TABLE = "stock"; @@ -38,17 +40,14 @@ class StockRowProducer extends AbstractRowProducer { s_data """; - private final long warehouseId; - StockRowProducer(DataLoadStatus status, long warehouseId, long rowCount) { super(TABLE, COLUMNS, rowCount, status::incStock); this.warehouseId = warehouseId; } @Override - String createRow(long rowIndex) { - return String.join( - ",", + List createRowsAsList(long rowIndex) { + return Arrays.asList( ImmutableList.of( getId(rowIndex), String.valueOf(warehouseId), diff --git a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/WarehouseRowProducer.java b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/WarehouseRowProducer.java index 7a05ae2c0..bc99c12c5 100644 --- a/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/WarehouseRowProducer.java +++ b/benchmarks/tpcc/src/main/java/com/google/cloud/pgadapter/tpcc/dataloader/WarehouseRowProducer.java @@ -14,6 +14,8 @@ package com.google.cloud.pgadapter.tpcc.dataloader; import com.google.common.collect.ImmutableList; +import java.util.Arrays; +import java.util.List; class WarehouseRowProducer extends AbstractRowProducer { private static final String TABLE = "warehouse"; @@ -25,9 +27,8 @@ class WarehouseRowProducer extends AbstractRowProducer { } @Override - String createRow(long rowIndex) { - return String.join( - ",", + List createRowsAsList(long rowIndex) { + return Arrays.asList( ImmutableList.of( getId(rowIndex), getRandomName(), diff --git a/benchmarks/tpcc/src/main/resources/schema_googlesql.sql b/benchmarks/tpcc/src/main/resources/schema_googlesql.sql new file mode 100644 index 000000000..919a5384f --- /dev/null +++ b/benchmarks/tpcc/src/main/resources/schema_googlesql.sql @@ -0,0 +1,151 @@ +SET AUTOCOMMIT = FALSE; +START BATCH DDL; + +CREATE TABLE IF NOT EXISTS warehouse ( + w_id INT64 not null, + w_name STRING(10), + w_street_1 STRING(20), + w_street_2 STRING(20), + w_city STRING(20), + w_state STRING(2), + w_zip STRING(9), + w_tax NUMERIC, + w_ytd NUMERIC, +) primary key (w_id); + +create table IF NOT EXISTS district ( + d_id INT64 not null, + w_id INT64 not null, + d_name STRING(10), + d_street_1 STRING(20), + d_street_2 STRING(20), + d_city STRING(20), + d_state STRING(2), + d_zip STRING(9), + d_tax NUMERIC, + d_ytd NUMERIC, + d_next_o_id INT64, +) primary key (w_id, d_id); + +-- CUSTOMER TABLE + +create table IF NOT EXISTS customer ( + c_id INT64 not null, + d_id INT64 not null, + w_id INT64 not null, + c_first STRING(16), + c_middle STRING(2), + c_last STRING(16), + c_street_1 STRING(20), + c_street_2 STRING(20), + c_city STRING(20), + c_state STRING(2), + c_zip STRING(9), + c_phone STRING(16), + c_since TIMESTAMP, + c_credit STRING(2), + c_credit_lim INT64, + c_discount NUMERIC, + c_balance NUMERIC, + c_ytd_payment NUMERIC, + c_payment_cnt INT64, + c_delivery_cnt INT64, + c_data STRING(MAX), +) PRIMARY KEY(w_id, d_id, c_id); + +-- HISTORY TABLE + +create table IF NOT EXISTS history ( + c_id INT64, + d_id INT64, + w_id INT64, + h_d_id INT64, + h_w_id INT64, + h_date TIMESTAMP, + h_amount NUMERIC, + h_data STRING(24), +) PRIMARY KEY(c_id, d_id, w_id, h_d_id, h_w_id, h_date); + +create table IF NOT EXISTS orders ( + o_id INT64 not null, + d_id INT64 not null, + w_id INT64 not null, + c_id INT64 not null, + o_entry_d TIMESTAMP, + o_carrier_id INT64, + o_ol_cnt INT64, + o_all_local INT64, +) PRIMARY KEY(w_id, d_id, c_id, o_id); + +-- NEW_ORDER table + +create table IF NOT EXISTS new_orders ( + o_id INT64 not null, + c_id INT64 not null, + d_id INT64 not null, + w_id INT64 not null, +) PRIMARY KEY(w_id, d_id, c_id, o_id); + +create table IF NOT EXISTS order_line ( + o_id INT64 not null, + c_id INT64 not null, + d_id INT64 not null, + w_id INT64 not null, + ol_number INT64 not null, + ol_i_id INT64, + ol_supply_w_id INT64, + ol_delivery_d TIMESTAMP, + ol_quantity INT64, + ol_amount NUMERIC, + ol_dist_info STRING(24), +) PRIMARY KEY(w_id, d_id, c_id, o_id, ol_number); + +-- STOCK table + +create table IF NOT EXISTS stock ( + s_i_id INT64 not null, + w_id INT64 not null, + s_quantity INT64, + s_dist_01 STRING(24), + s_dist_02 STRING(24), + s_dist_03 STRING(24), + s_dist_04 STRING(24), + s_dist_05 STRING(24), + s_dist_06 STRING(24), + s_dist_07 STRING(24), + s_dist_08 STRING(24), + s_dist_09 STRING(24), + s_dist_10 STRING(24), + s_ytd NUMERIC, + s_order_cnt INT64, + s_remote_cnt INT64, + s_data STRING(50), +) PRIMARY KEY(w_id, s_i_id); + +create table IF NOT EXISTS item ( + i_id INT64 not null, + i_im_id INT64, + i_name STRING(24), + i_price NUMERIC, + i_data STRING(50), +) PRIMARY KEY(i_id); + +CREATE INDEX idx_customer ON customer (w_id,d_id,c_last,c_first); +CREATE INDEX idx_orders ON orders (w_id,d_id,c_id,o_id); +CREATE INDEX fkey_stock_2 ON stock (s_i_id); +CREATE INDEX fkey_order_line_2 ON order_line (ol_supply_w_id,ol_i_id); +CREATE INDEX fkey_history_1 ON history (w_id,d_id,c_id); +CREATE INDEX fkey_history_2 ON history (h_w_id,h_d_id ); + +ALTER TABLE new_orders ADD CONSTRAINT fkey_new_orders_1_ FOREIGN KEY(w_id,d_id,c_id,o_id) REFERENCES orders(w_id,d_id,c_id,o_id); +ALTER TABLE orders ADD CONSTRAINT fkey_orders_1_ FOREIGN KEY(w_id,d_id,c_id) REFERENCES customer(w_id,d_id,c_id); +ALTER TABLE customer ADD CONSTRAINT fkey_customer_1_ FOREIGN KEY(w_id,d_id) REFERENCES district(w_id,d_id); +ALTER TABLE history ADD CONSTRAINT fkey_history_1_ FOREIGN KEY(w_id,d_id,c_id) REFERENCES customer(w_id,d_id,c_id); +ALTER TABLE history ADD CONSTRAINT fkey_history_2_ FOREIGN KEY(h_w_id,h_d_id) REFERENCES district(w_id,d_id); +ALTER TABLE district ADD CONSTRAINT fkey_district_1_ FOREIGN KEY(w_id) REFERENCES warehouse(w_id); +ALTER TABLE order_line ADD CONSTRAINT fkey_order_line_1_ FOREIGN KEY(w_id,d_id,c_id,o_id) REFERENCES orders(w_id,d_id,c_id,o_id); +ALTER TABLE order_line ADD CONSTRAINT fkey_order_line_2_ FOREIGN KEY(ol_supply_w_id,ol_i_id) REFERENCES stock(w_id,s_i_id); +ALTER TABLE stock ADD CONSTRAINT fkey_stock_1_ FOREIGN KEY(w_id) REFERENCES warehouse(w_id); +ALTER TABLE stock ADD CONSTRAINT fkey_stock_2_ FOREIGN KEY(s_i_id) REFERENCES item(i_id); + +RUN BATCH;