feat: use canbench for benchmarks

.github/workflows/canbench_ci_post_run_benchmark.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -Eexuo pipefail
+
+if [ "$EXIT_STATUS" -eq 1 ]; then
+  exit 1
+fi

.github/workflows/canbench_ci_run_benchmark.sh

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+set -Eexuo pipefail
+
+# Script that runs `canbench` at a given directory and outputs a comment
+# that is intended to be posted on the pull request.
+
+# Path to run `canbench` from.
+CANISTER_PATH=$1
+
+# Must match the file specified in the github action.
+COMMENT_MESSAGE_PATH=/tmp/canbench_comment_message.txt
+
+# Github CI is expected to have the main branch checked out in this folder.
+MAIN_BRANCH_DIR=_canbench_main_branch
+
+CANBENCH_OUTPUT=/tmp/canbench_output.txt
+
+CANBENCH_RESULTS_FILE="$CANISTER_PATH/canbench_results.yml"
+MAIN_BRANCH_RESULTS_FILE="$MAIN_BRANCH_DIR/$CANBENCH_RESULTS_FILE"
+
+# Install canbench
+cargo install canbench
+
+# Verify that canbench results are available.
+if [ ! -f "$CANBENCH_RESULTS_FILE" ]; then
+    echo "$CANBENCH_RESULTS_FILE not found. Did you forget to run \`canbench --persist\`?";
+    exit 1
+fi
+
+# Detect if canbench results file is up to date.
+pushd "$CANISTER_PATH"
+canbench --less-verbose > $CANBENCH_OUTPUT
+if grep -q "(regress\|(improved by \|(new)" "$CANBENCH_OUTPUT"; then
+  UPDATED_MSG="**\`$CANBENCH_RESULTS_FILE\` is not up to date ❌**
+  If the performance change is expected, run \`canbench --persist\` to save the updated benchmark results.";
+
+  # canbench results file not up to date. Fail the job.
+  echo "EXIT_STATUS=1" >> "$GITHUB_ENV"
+else
+  UPDATED_MSG="**\`$CANBENCH_RESULTS_FILE\` is up to date ✅**";
+
+  # canbench results file is up to date. The job succeeds.
+  echo "EXIT_STATUS=0" >> "$GITHUB_ENV"
+fi
+popd
+
+
+echo "# \`canbench\` 🏋" > $COMMENT_MESSAGE_PATH
+
+# Detect if there are performance changes relative to the main branch.
+if [ -f "$MAIN_BRANCH_RESULTS_FILE" ]; then
+  # Move the results of the main branch into the current branch.
+  mv "$MAIN_BRANCH_RESULTS_FILE" "$CANBENCH_RESULTS_FILE"
+
+  # Run canbench to compare result to main branch.
+  pushd "$CANISTER_PATH"
+  set +e
+  canbench --less-verbose > $CANBENCH_OUTPUT
+  RES=$?
+  set -e
+  popd
+
+  if [ "$RES" -eq 0 ]; then
+    if grep -q "(regress\|(improved by" "${CANBENCH_OUTPUT}"; then
+      echo "**Significant performance change detected! ⚠️**
+      " >> $COMMENT_MESSAGE_PATH;
+    else
+      echo "**No significant performance changes detected ✅**
+      " >> $COMMENT_MESSAGE_PATH
+    fi
+  else
+    echo "Failed to run \`canbench\` against main branch ⚠️
+    " >> $COMMENT_MESSAGE_PATH
+  fi
+fi
+
+## Add the output of canbench to the file.
+{
+  echo "$UPDATED_MSG"
+  echo ""
+  echo "\`\`\`"
+  cat "$CANBENCH_OUTPUT"
+  echo "\`\`\`"
+} >> $COMMENT_MESSAGE_PATH
+
+# Output the comment to stdout.
+cat $COMMENT_MESSAGE_PATH

.github/workflows/ci.yml

@@ -41,7 +41,7 @@ jobs:
           RUST_BACKTRACE: 1
 
   examples:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v2
@@ -72,33 +72,56 @@ jobs:
           bash examples/test.sh
 
   benchmark:
-    runs-on: ubuntu-20.04
-
+    runs-on: ubuntu-latest
+    needs: build
+    env:
+      PROJECT_DIR: .
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/cache@v3
+      - name: Checkout current PR
+        uses: actions/checkout@v4
+
+      - name: Checkout main branch
+        uses: actions/checkout@v4
+        with:
+          ref: main
+          path: _canbench_main_branch
+
+      - name: Cache Cargo
+        uses: actions/cache@v2
         with:
           path: |
             ~/.cargo/registry
             ~/.cargo/git
             target
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-1
+          key: ${{ matrix.build }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: |
+            ${{ matrix.build }}-cargo-
 
       - name: Install Rust
         run: |
           rustup update ${{ matrix.rust }} --no-self-update
           rustup default ${{ matrix.rust }}
           rustup target add wasm32-unknown-unknown
 
-      - name: Run benchmark test
+      - name: Benchmark
+        run: |
+          bash .github/workflows/canbench_ci_run_benchmark.sh $PROJECT_DIR
+
+      - name: Post comment
+        uses: thollander/actions-comment-pull-request@v2
+        with:
+          filePath: /tmp/canbench_comment_message.txt
+          comment_tag: ${{ env.PROJECT_DIR }}
+
+      - name: Pass or fail
         run: |
-          bash .github/workflows/benchmark_ci.sh
+          bash .github/workflows/canbench_ci_post_run_benchmark.sh
 
   checks-pass:
     # Always run this job!
     if: always()
     needs: [build, examples, benchmark]
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - name: check build result
         if: ${{ needs.build.result != 'success' }}