Report benchmark performance per Pull Request

This commit adds github-action-benchmark which
can report performance change between current
commit and previous commit.

Signed-off-by: Boxuan Li <liboxuan@connect.hku.hk>

.github/workflows/ci-benchmark.yml

@@ -0,0 +1,82 @@
+# Copyright 2022 JanusGraph Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: CI Benchmark
+
+on:
+  push:
+    paths-ignore:
+    - 'docs/**'
+    - '.github/workflows/ci-docs.yml'
+    - 'requirements.txt'
+    - 'docs.Dockerfile'
+    - '*.md'
+
+env:
+  BUILD_MAVEN_OPTS: "-DskipTests=true --batch-mode --also-make"
+
+permissions:
+  contents: write
+
+jobs:
+  build-all:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+      - uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+      - run: mvn clean install --projects janusgraph-all -Pjanusgraph-cache -Dmaven.javadoc.skip=true ${{ env.BUILD_MAVEN_OPTS }}
+
+  benchmark:
+    name: Performance regression check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+      - uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+      - run: mvn clean install -Pjanusgraph-benchmark ${{ env.BUILD_MAVEN_OPTS }} -Dgpg.skip=true
+      - run: mvn verify --projects janusgraph-benchmark
+
+      # Download previous benchmark result from cache (if exists)
+      - name: Download previous benchmark data
+        uses: actions/cache@v1
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark
+
+      # Run `github-action-benchmark` action
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: 'customSmallerIsBetter'
+          output-file-path: janusgraph-benchmark/benchmark.json
+          external-data-json-path: ./cache/benchmark-data.json
+          fail-on-alert: false
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          comment-on-alert: true
+          comment-always: true

.github/workflows/ci-release.yml

@@ -86,23 +86,6 @@ jobs:
           name: distribution-builds
           path: janusgraph-dist/target/janusgraph-*.zip
 
-  benchmark:
-    runs-on: ubuntu-20.04
-    needs: build-all
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
-        with:
-          path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
-          restore-keys: |
-            ${{ runner.os }}-maven-
-      - uses: actions/setup-java@v1
-        with:
-          java-version: 1.8
-      - run: mvn clean install -Pjanusgraph-benchmark ${{ env.BUILD_MAVEN_OPTS }} -Dgpg.skip=true
-      - run: mvn verify --projects janusgraph-benchmark
-
   tp-tests:
     runs-on: ubuntu-20.04
     if: "github.event_name == 'push' && contains(github.event.head_commit.message, '[tp-tests]') || github.event_name == 'pull_request' && contains(github.event.pull_request.title, '[tp-tests]') || github.event_name == 'schedule'"

janusgraph-benchmark/src/main/java/org/janusgraph/BenchmarkRunner.java

@@ -14,14 +14,27 @@
 
 package org.janusgraph;
 
+import org.apache.tinkerpop.shaded.jackson.databind.ObjectMapper;
+import org.openjdk.jmh.results.Result;
+import org.openjdk.jmh.results.RunResult;
 import org.openjdk.jmh.runner.Runner;
 import org.openjdk.jmh.runner.RunnerException;
 import org.openjdk.jmh.runner.options.ChainedOptionsBuilder;
 import org.openjdk.jmh.runner.options.OptionsBuilder;
 import org.openjdk.jmh.runner.options.TimeValue;
 
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 public class BenchmarkRunner {
-    public static void main(String[] args) throws RunnerException {
+    public static void main(String[] args) throws RunnerException, IOException {
         final ChainedOptionsBuilder builder = new OptionsBuilder()
             .forks(1)
             .measurementTime(TimeValue.seconds(5))
@@ -36,6 +49,34 @@ public static void main(String[] args) throws RunnerException {
             builder.include(".*Benchmark");
         }
         builder.exclude(".*StaticArrayEntryListBenchmark");
-        new Runner(builder.build()).run();
+        Collection<RunResult> results = new Runner(builder.build()).run();
+        Map<String, Double> metrics = new HashMap<>();
+        for (RunResult result : results) {
+            Result primaryResult = result.getPrimaryResult();
+            String benchmark = result.getParams().getBenchmark();
+            double score = primaryResult.getScore();
+            String unit = primaryResult.getScoreUnit();
+            if (!"ms/op".equals(unit)) {
+                throw new IllegalArgumentException("Please use ms/op as measurement for benchmark " + benchmark);
+            }
+            // we sum up results with different params against same benchmark
+            metrics.put(benchmark, metrics.getOrDefault(benchmark, 0d) + score);
+        }
+        List<Map<String, Object>> outputs = new ArrayList<>();
+        for (Map.Entry<String, Double> entry : metrics.entrySet()) {
+            outputs.add(new HashMap() {{
+                put("name", entry.getKey());
+                put("value", entry.getValue());
+                put("unit", "ms/op");
+            }});
+        }
+        String workspace = System.getenv("${GITHUB_WORKSPACE}");
+        File file = new File(workspace, "benchmark.json");
+        System.out.println("writing to " + file.getAbsolutePath());
+        ObjectMapper objectMapper = new ObjectMapper();
+        String json = objectMapper.writeValueAsString(outputs);
+        BufferedWriter writer = new BufferedWriter(new FileWriter(file));
+        writer.write(json);
+        writer.close();
     }
 }

janusgraph-benchmark/src/main/java/org/janusgraph/JanusGraphSpeedBenchmark.java

@@ -25,13 +25,20 @@
 import org.janusgraph.diskstorage.configuration.ModifiableConfiguration;
 import org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration;
 import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
 import org.openjdk.jmh.annotations.Param;
 import org.openjdk.jmh.annotations.Scope;
 import org.openjdk.jmh.annotations.Setup;
 import org.openjdk.jmh.annotations.State;
 import org.openjdk.jmh.annotations.TearDown;
 
+import java.util.concurrent.TimeUnit;
+
 @State(Scope.Thread)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
 public class JanusGraphSpeedBenchmark {
 
     @Param({ "1000", "10000", "100000" })