Track PR metrics with Bencher

.github/workflows/metrics.yml

@@ -19,6 +19,10 @@ jobs:
       fail-fast: false
       matrix:
         backend: ["postgres", "sqlite", "mysql"]
+    env:
+      BENCHER_PROJECT: diesel
+      BENCHER_BRANCH: master
+      BENCHER_ADAPTER: rust_criterion
     steps:
       - name: Checkout sources
         uses: actions/checkout@v4
@@ -61,7 +65,7 @@ jobs:
 
       - name: Run Benchmarks (Postgres)
         if: matrix.backend == 'postgres'
-        run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "postgres sqlx-bench sqlx/postgres rust_postgres futures sea-orm sea-orm/sqlx-postgres criterion/async_tokio quaint quaint/postgresql quaint/serde-support serde diesel-async diesel-async/postgres wtx"
+        run: bencher run 'cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "postgres sqlx-bench sqlx/postgres rust_postgres futures sea-orm sea-orm/sqlx-postgres criterion/async_tokio quaint quaint/postgresql quaint/serde-support serde diesel-async diesel-async/postgres wtx"'
 
       - name: Run Benchmarks (Sqlite)
         if: matrix.backend == 'sqlite'

.github/workflows/benches.yml → .github/workflows/run_benches.yml

@@ -1,13 +1,12 @@
 on:
   pull_request:
-    types:
-      - labeled
+    types: [labeled, opened, reopened, synchronize]
 
-name: Benchmarks
+name: Run and Cache Benchmarks
 
 jobs:
-  benchmarks:
-    if: github.event.label.name == 'run-benchmarks'
+  run_benchmarks:
+    if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-benchmarks')
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -50,16 +49,35 @@ jobs:
       - name: Install critcmp
         run: cargo +stable install critcmp
 
-      - name: Benchmark changes
-        run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{matrix.backend}}" -- --save-baseline changes
+      - name: Benchmark PR ${{ matrix.backend }}
+        run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{ matrix.backend }}" -- --save-baseline changes > pr_${{ matrix.backend }}.txt
 
-      - name: Checkout master
-        run: |
-          git fetch origin
-          git reset --hard origin/master
+      - name: Upload PR ${{ matrix.backend }} Benchmark Results
+        uses: actions/upload-artifact@v4
+        with:
+          name: pr_${{ matrix.backend }}.txt
+          path: ./pr_${{ matrix.backend }}.txt
+
+      - name: Checkout base branch
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
+          repository: ${{ github.event.pull_request.base.repo.full_name }}
+
+      - name: Benchmark base ${{ matrix.backend }}
+        run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{ matrix.backend }}" -- --save-baseline master > base_${{ matrix.backend }}.txt
 
-      - name: Benchmark master
-        run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{matrix.backend}}" -- --save-baseline master
+      - name: Upload base ${{ matrix.backend }} Benchmark Results
+        uses: actions/upload-artifact@v4
+        with:
+          name: base_${{ matrix.backend }}.txt
+          path: ./base_${{ matrix.backend }}.txt
+
+      - name: Upload GitHub Event
+        uses: actions/upload-artifact@v4
+        with:
+          name: event_${{ matrix.backend }}.json
+          path: ${{ github.event_path }}
 
       - name: Critcmp
         run: |
@@ -70,19 +88,3 @@ jobs:
           echo '```' >> $GITHUB_STEP_SUMMARY
           critcmp master changes >> $GITHUB_STEP_SUMMARY
           echo '```' >> $GITHUB_STEP_SUMMARY
-
-      # This does not work due to github not allowing to post comments from forked repos
-      # - name: Post the output as comment
-      #   uses: actions/github-script@v3
-      #   with:
-      #     github-token: ${{secrets.GITHUB_TOKEN}}
-      #     script: |
-      #       const fs = require('fs');
-      #       const data = fs.readFileSync('diesel_bench/output.txt', 'utf8');
-
-      #       github.issues.createComment({
-      #         issue_number: context.issue.number,
-      #         owner: context.repo.owner,
-      #         repo: context.repo.repo,
-      #         body: data
-      #       })

.github/workflows/track_benches.yml

@@ -0,0 +1,100 @@
+on:
+  workflow_run:
+    workflows: [Run and Cache Benchmarks]
+    types:
+      - completed
+
+name: Track Benchmarks
+
+jobs:
+  track_benchmarks:
+    if: github.event.workflow_run.conclusion == 'success'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        backend: ["postgres", "sqlite", "mysql"]
+    env:
+      BENCHER_PROJECT: diesel
+      BENCHER_ADAPTER: rust_criterion
+      BENCHER_TESTBED: ubuntu-latest-${{ matrix.backend }}
+      PR_BENCHMARK_RESULTS: pr_${{ matrix.backend }}.txt
+      BASE_BENCHMARK_RESULTS: base_${{ matrix.backend }}.txt
+      GITHUB_EVENT: event_${{ matrix.backend }}.json
+      # This is the confidence interval for the t-test Threshold
+      # Adjust this value to lower to make the test more sensitive to changes
+      # Adjust this value to higher to make the test less sensitive to changes
+      # https://bencher.dev/docs/explanation/thresholds/#t-test-threshold-upper-boundary
+      UPPER_BOUNDARY: 0.98
+    steps:
+      - name: Download Benchmark Results
+        uses: actions/github-script@v6
+        with:
+          script: |
+            async function downloadArtifact(artifactName) {
+              let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                run_id: context.payload.workflow_run.id,
+              });
+              let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
+                return artifact.name == artifactName
+              })[0];
+              if (!matchArtifact) {
+                core.setFailed(`Failed to find artifact: ${artifactName}`);
+              }
+              let download = await github.rest.actions.downloadArtifact({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                artifact_id: matchArtifact.id,
+                archive_format: 'zip',
+              });
+              let fs = require('fs');
+              fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/${artifactName}.zip`, Buffer.from(download.data));
+            }
+            await downloadArtifact(process.env.PR_BENCHMARK_RESULTS);
+            await downloadArtifact(process.env.BASE_BENCHMARK_RESULTS);
+            await downloadArtifact(process.env.GITHUB_EVENT);
+      - name: Unzip Benchmark Results
+        run: |
+          unzip $PR_BENCHMARK_RESULTS.zip
+          unzip $BASE_BENCHMARK_RESULTS.zip
+          unzip $GITHUB_EVENT.zip
+      - name: Export GitHub Event Data
+        uses: actions/github-script@v6
+        with:
+          script: |
+            let fs = require('fs');
+            let githubEvent = JSON.parse(fs.readFileSync("event.json", {encoding: 'utf8'}));
+            console.log(githubEvent);
+            core.exportVariable("PR_HEAD", `${githubEvent.pull_request.head.ref}-${githubEvent.pull_request.head.sha.slice(0, 8)}`);
+            core.exportVariable("PR_ID", `${githubEvent.pull_request.head.ref}/${process.env.BENCHER_TESTBED}/${process.env.BENCHER_ADAPTER}`);
+            core.exportVariable("PR_NUMBER", githubEvent.number);
+      - uses: bencherdev/bencher@main
+      - name: Track base Benchmarks
+        run: |
+          bencher run \
+          --if-branch '${{ env.PR_HEAD }}' \
+          --else-branch \
+          --token "${{ secrets.BENCHER_API_TOKEN }}" \
+          --file "$BASE_BENCHMARK_RESULTS"
+      - name: Create PR threshold
+        run: |
+          bencher threshold create \
+          --project "$BENCHER_PROJECT" \
+          --branch '${{ env.PR_HEAD }}' \
+          --testbed "$BENCHER_TESTBED" \
+          --measure latency \
+          --test t \
+          --upper-boundary ${{ env.UPPER_BOUNDARY }} \
+          --token "${{ secrets.BENCHER_API_TOKEN }}"
+      - name: Track PR Benchmarks
+        run: |
+          bencher run \
+          --branch '${{ env.PR_HEAD }}' \
+          --token "${{ secrets.BENCHER_API_TOKEN }}" \
+          --ci-id '${{ env.PR_ID }}' \
+          --ci-number '${{ env.PR_NUMBER }}' \
+          --github-actions "${{ secrets.GITHUB_TOKEN }}" \
+          --err \
+          --file "$PR_BENCHMARK_RESULTS"