Skip to content
Permalink
Browse files
feat: add jmh benchmark module (#686)
* add jmh benchmark

* fix typo for jar name

* update README.md with additional instructions to run Benchmarks

* update benchmarkMode to AverageTime and reduce iterations

Co-authored-by: stephwang <stephwang@google.com>
  • Loading branch information
Praful Makani and stephaniewang526 committed Aug 25, 2020
1 parent f0e67d8 commit a0e760fbb1f5a9b169045ceba2dcf682d4736995
Showing with 199 additions and 0 deletions.
  1. +20 −0 benchmark/README.md
  2. +77 −0 benchmark/pom.xml
  3. +96 −0 benchmark/src/main/java/com.google.cloud.bigquery/QueryBenchmark.java
  4. +6 −0 pom.xml
@@ -0,0 +1,20 @@
Benchmarking
============

To build all of the benchmarks:
```
# Run from benchmark directory
cd benchmark
mvn clean install
```

Set `GOOGLE_APPLICATION_CREDENTIALS`:
```
export GOOGLE_APPLICATION_CREDENTIALS=path/to/service_account.json
```

To run a benchmark jar, run the following command
```
# Run from project root directory
java -jar target/benchmark.jar
```
@@ -0,0 +1,77 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>benchmark</artifactId>
<parent>
<artifactId>google-cloud-bigquery-parent</artifactId>
<groupId>com.google.cloud</groupId>
<version>1.116.11-SNAPSHOT</version>
</parent>

<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<jmh.version>1.23</jmh.version>
<uberjar.name>benchmark</uberjar.name>
</properties>

<dependencies>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-bigquery</artifactId>
</dependency>
<dependency>
<groupId>org.openjdk.jmh</groupId>
<artifactId>jmh-core</artifactId>
<version>${jmh.version}</version>
</dependency>
<dependency>
<groupId>org.openjdk.jmh</groupId>
<artifactId>jmh-generator-annprocess</artifactId>
<version>${jmh.version}</version>
<scope>provided</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.0</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.1</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<finalName>${uberjar.name}</finalName>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>org.openjdk.jmh.Main</mainClass>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
</transformers>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>module-info.class</exclude>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
@@ -0,0 +1,96 @@
/*
* Copyright 2020 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.google.cloud.bigquery;

import java.util.List;
import java.util.concurrent.TimeUnit;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.Warmup;
import org.openjdk.jmh.infra.Blackhole;

@Fork(value = 1)
@BenchmarkMode(Mode.AverageTime)
@Warmup(iterations = 5)
@Measurement(iterations = 5)
@State(Scope.Benchmark)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
public class QueryBenchmark {

private static final String NYCYELLOWLIMIT1K = "SELECT * FROM `nyc-tlc.yellow.trips` LIMIT 10000";
private static final String NYCYELLOWLIMIT10K =
"SELECT * FROM `nyc-tlc.yellow.trips` LIMIT 100000";
private static final String NYCYELLOWLIMIT100K =
"SELECT * FROM `nyc-tlc.yellow.trips` LIMIT 1000000";
private static final String WIKISAMPLESORDEREDLIMIT1K =
"SELECT title FROM `bigquery-public-data.samples.wikipedia` ORDER BY title LIMIT 1000";
private static final String CURRENTTIMESTAMP = "SELECT CURRENT_TIMESTAMP() as ts";
private static final String SESSIONUSER = "SELECT SESSION_USER() as ts";
private static final String LITERALS = "SELECT 1 as i, 3.14 as pi";
private static final String INVALIDQUERY =
"CREATE OR REPLACE SELECT * FROM UPDATE TABLE SET `nyc-tlc.yellow.trips`";

private BigQuery bigquery;

@Setup
public void setUp() {
this.bigquery = BigQueryOptions.getDefaultInstance().getService();
}

@State(Scope.Benchmark)
public static class QueryParams {

@Param({
NYCYELLOWLIMIT1K,
NYCYELLOWLIMIT10K,
NYCYELLOWLIMIT100K,
WIKISAMPLESORDEREDLIMIT1K,
CURRENTTIMESTAMP,
SESSIONUSER,
LITERALS,
INVALIDQUERY
})
public String queries;
}

private void queryPerform(String queries, Blackhole blackhole) throws Exception {
TableResult result =
bigquery.query(QueryJobConfiguration.newBuilder(queries).setUseLegacySql(false).build());
for (List<FieldValue> row : result.iterateAll()) {
blackhole.consume(row.size());
}
while (result.hasNextPage()) {
result = result.getNextPage();
for (List<FieldValue> row : result.iterateAll()) {
blackhole.consume(row.size());
}
}
}

@Benchmark
public void query(QueryParams queryParams, Blackhole blackhole) throws Exception {
queryPerform(queryParams.queries, blackhole);
}
}
@@ -215,5 +215,11 @@
<module>samples</module>
</modules>
</profile>
<profile>
<id>benchmark</id>
<modules>
<module>benchmark</module>
</modules>
</profile>
</profiles>
</project>

0 comments on commit a0e760f

Please sign in to comment.