feat: run benchmarks for ACIR proving (#6155)

Resolves #6060 
---------

Co-authored-by: PhilWindle <philip.windle@gmail.com>
Co-authored-by: ludamad <adam.domurad@gmail.com>
Co-authored-by: Santiago Palladino <santiago@aztecprotocol.com>

.github/workflows/ci.yml

@@ -153,8 +153,39 @@ jobs:
               --no-output \
               +${{ matrix.test }}
 
+  acir-bench:
+    runs-on: ubuntu-latest
+    needs: setup
+    steps:
+      - uses: actions/checkout@v4
+        with: { ref: "${{ env.GIT_COMMIT }}" }
+      - uses: ./.github/ci-setup-action
+      - name: Setup and Test
+        uses: ./.github/ensure-tester-with-images
+        timeout-minutes: 40
+        with:
+          runner_type: 16core-tester-x86
+          builder_type: builder-x86
+          # these are copied to the tester and expected by the earthly command below
+          # if they fail to copy, it will try to build them on the tester and fail
+          builder_images_to_copy: aztecprotocol/barretenberg-acir-benches:${{ env.GIT_COMMIT }}
+          # command to produce the images in case they don't exist
+          builder_command: cd noir && ../scripts/earthly-ci +export-bench-acir-bb
+          run: |
+            set -eux
+            cd ./noir/
+            export FORCE_COLOR=1
+            export EARTHLY_BUILD_ARGS="${{ env.EARTHLY_BUILD_ARGS }}"
+            ../scripts/earthly-ci -P \
+              --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
+              --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
+              --no-output \
+              +bench-publish-acir-bb         
+
   bench-summary:
-    needs: bench-e2e
+    needs: 
+      - acir-bench
+      - bench-e2e
     runs-on: ${{ inputs.username || github.actor }}-x86
     steps:
       - uses: actions/checkout@v4
@@ -181,24 +212,6 @@ jobs:
         working-directory: ./yarn-project/scripts
         run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} +bench-comment
 
-  noir-format:
-    needs: setup
-    runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
-    steps:
-      - uses: actions/checkout@v4
-        with: { ref: "${{ env.GIT_COMMIT }}" }
-      - uses: ./.github/ci-setup-action
-        with:
-          concurrency_key: noir-format-x86
-      - name: "Format Noir"
-        working-directory: ./noir/
-        timeout-minutes: 40
-        run: earthly-ci --no-output ./+format
-      - name: "Format noir-projects"
-        working-directory: ./noir-projects/
-        timeout-minutes: 40
-        run: earthly-ci --no-output ./+format
-
   # barretenberg (prover) native and AVM (public VM) tests
   # only ran on x86 for resource reasons (memory intensive)
   bb-native-tests:
@@ -232,6 +245,24 @@ jobs:
         timeout-minutes: 40
         run: earthly-ci --no-output ./+test
 
+  noir-format:
+    needs: setup
+    runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
+    steps:
+      - uses: actions/checkout@v4
+        with: { ref: "${{ env.GIT_COMMIT }}" }
+      - uses: ./.github/ci-setup-action
+        with:
+          concurrency_key: noir-format-x86
+      - name: "Format Noir"
+        working-directory: ./noir/
+        timeout-minutes: 40
+        run: earthly-ci --no-output ./+format
+      - name: "Format noir-projects"
+        working-directory: ./noir-projects/
+        timeout-minutes: 40
+        run: earthly-ci --no-output ./+format
+
   noir-test:
     needs: setup
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
@@ -468,6 +499,7 @@ jobs:
       - setup
       - e2e
       - bb-native-tests
+      - acir-bench
       - bb-bench
       - yarn-project-formatting
       - yarn-project-test

Earthfile

@@ -60,4 +60,4 @@ base-log-uploader:
         unzip awscliv2.zip && \
         ./aws/install --bin-dir /usr/local/bin --install-dir /usr/local/aws-cli --update && \
         rm -rf aws awscliv2.zip
-    COPY +scripts/scripts /usr/src/scripts
+    COPY +scripts/scripts /usr/src/scripts

barretenberg/acir_tests/bench_acir_tests.sh

@@ -1,8 +1,14 @@
 #!/usr/bin/env bash
 
+cd "$(dirname "$0")"
+
 TEST_NAMES=("$@")
 THREADS=(1 4 16 32 64)
-BENCHMARKS=$(mktemp)
+BENCHMARKS=$LOG_FILE
+
+if [[ -z "${LOG_FILE}" ]]; then
+    BENCHMARKS=$(mktemp)
+fi
 
 if [ "${#TEST_NAMES[@]}" -eq 0 ]; then
     TEST_NAMES=(sha256 ecdsa_secp256k1 ecdsa_secp256r1 schnorr double_verify_proof)
@@ -64,4 +70,6 @@ function genthreadheaders(t, len,   res) {
 }
 '
 
-rm $BENCHMARKS
+if [[ -z "${LOG_FILE}" ]]; then
+    rm $BENCHMARKS
+fi

barretenberg/cpp/src/barretenberg/common/benchmark.hpp

@@ -94,7 +94,7 @@ template <typename T, typename... Args> void write_benchmark(const std::string&
     }
     std::ostringstream oss;
     oss << "{\"timestamp\": \"" << getCurrentTimestamp() << "\", "
-        << "\"name\": \"" << name << "\", "
+        << "\"eventName\": \"" << name << "\", "
         << "\"type\": \"" << TypeTraits<T>::type << "\", "
         << "\"value\": " << value << ", "
         << "\"threads\": " << env_hardware_concurrency();

noir/Earthfile

@@ -216,6 +216,43 @@ build-acir-tests:
     RUN ./rebuild.sh true
     SAVE ARTIFACT /usr/src/noir-repo/test_programs/acir_artifacts/*
 
+barretenberg-acir-benches-bb:
+  FROM ubuntu:noble
+  RUN apt update && \
+      apt install -y curl git jq unzip
+  COPY ../barretenberg/cpp/+preset-clang-assert/bin/bb /usr/src/barretenberg/cpp/build/bin/bb
+  COPY ../barretenberg/+acir-tests/ /usr/src/barretenberg/acir_tests
+  COPY +build-acir-tests/ /usr/src/acir_artifacts
+
+  ENV TEST_SRC=/usr/src/acir_artifacts
+
+  WORKDIR /usr/src/barretenberg/acir_tests
+
+export-bench-acir-bb:
+  ARG EARTHLY_GIT_HASH
+  FROM +barretenberg-acir-benches-bb
+  SAVE IMAGE aztecprotocol/barretenberg-acir-benches:$EARTHLY_GIT_HASH
+
+bench-publish-acir-bb:
+  ARG PULL_REQUEST
+  ARG BRANCH
+  ARG COMMIT_HASH
+  LOCALLY
+
+  # Let docker compose know about the pushed tags above
+  ENV AZTEC_DOCKER_TAG=$(git rev-parse HEAD)
+  # Optimize to not cause serial behavior if image already exists
+  IF ! docker image ls --format '{{.Repository}}:{{.Tag}}' | grep "aztecprotocol/barretenberg-acir-benches:$AZTEC_DOCKER_TAG"
+    WAIT
+      BUILD +export-bench-acir-bb
+    END
+  END
+
+  RUN mkdir -p ./log
+  RUN docker run -v "$(pwd)/log":/log -e LOG_FILE=/log/bench-acir.jsonl --rm aztecprotocol/barretenberg-acir-benches:$AZTEC_DOCKER_TAG ./bench_acir_tests.sh sha256
+
+  DO ../+UPLOAD_LOGS --PULL_REQUEST=$PULL_REQUEST --BRANCH=$BRANCH --COMMIT_HASH=$COMMIT_HASH
+
 barretenberg-acir-tests-bb:
     FROM ../build-images/+build
 

yarn-project/circuit-types/src/stats/metrics.ts

@@ -2,6 +2,7 @@ import { type StatsEventName } from './stats.js';
 
 /** How a metric is grouped in benchmarks: by block size, by length of chain processed, or by circuit name. */
 export type MetricGroupBy =
+  | 'threads'
   | 'block-size'
   | 'chain-length'
   | 'protocol-circuit-name'
@@ -25,6 +26,12 @@ export interface Metric {
 
 /** Metric definitions to track from benchmarks. */
 export const Metrics = [
+  {
+    name: 'proof_construction_time_sha256',
+    groupBy: 'threads',
+    description: 'Time needed to generate a proof of an ACIR program.',
+    events: ['proof_construction_time'],
+  },
   {
     name: 'l1_rollup_calldata_size_in_bytes',
     groupBy: 'block-size',

yarn-project/circuit-types/src/stats/stats.ts

@@ -1,3 +1,15 @@
+/** Stats associated with an ACIR proof generation.*/
+export type ProofConstructed = {
+  /** Name of the event for metrics purposes */
+  eventName: 'proof_construction_time';
+  /** Name of the program being proven */
+  acir_test: string;
+  /** Number of threads used for proving */
+  threads: number;
+  /** Time spent proving */
+  value: number;
+};
+
 /** Stats associated with an L2 block. */
 export type L2BlockStats = {
   /** Number of txs in the L2 block. */
@@ -255,6 +267,7 @@ export type TxAddedToPoolStats = {
 
 /** Stats emitted in structured logs with an `eventName` for tracking. */
 export type Stats =
+  | ProofConstructed
   | L1PublishStats
   | NodeSyncedChainHistoryStats
   | CircuitSimulationStats

yarn-project/scripts/Earthfile

@@ -65,5 +65,4 @@ bench-comment:
   RUN --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN \
     [ -f $BENCH_FOLDER/benchmark.json ] \
       && (cd /usr/src/yarn-project/scripts && AZTEC_BOT_COMMENTER_GITHUB_TOKEN=$AZTEC_BOT_COMMENTER_GITHUB_TOKEN yarn bench-comment) \
-      || echo "No benchmark file found in $BENCH_FOLDER"
-
+      || echo "No benchmark file found in $BENCH_FOLDER"

yarn-project/scripts/src/benchmarks/aggregate.ts

@@ -24,6 +24,7 @@ import {
   type MetricName,
   type NodeSyncedChainHistoryStats,
   type NoteProcessorCaughtUpStats,
+  type ProofConstructed,
   type Stats,
   type TreeInsertionStats,
   type TxAddedToPoolStats,
@@ -66,6 +67,13 @@ function append(
   results[metric]![bucket].push(numeric);
 }
 
+/** Processes an entry with event name 'acir-proof-generated' and updates results */
+function processAcirProofGenerated(entry: ProofConstructed, results: BenchmarkCollectedResults) {
+  if (entry.acir_test === 'sha256') {
+    append(results, `proof_construction_time_sha256`, entry.threads, entry.value);
+  }
+}
+
 /** Processes an entry with event name 'rollup-published-to-l1' and updates results */
 function processRollupPublished(entry: L1PublishStats, results: BenchmarkCollectedResults) {
   const bucket = entry.txCount;
@@ -240,6 +248,8 @@ function processTreeInsertion(entry: TreeInsertionStats, results: BenchmarkColle
 /** Processes a parsed entry from a log-file and updates results */
 function processEntry(entry: Stats, results: BenchmarkCollectedResults, fileName: string) {
   switch (entry.eventName) {
+    case 'proof_construction_time':
+      return processAcirProofGenerated(entry, results);
     case 'rollup-published-to-l1':
       return processRollupPublished(entry, results);
     case 'l2-block-handled':

yarn-project/scripts/src/benchmarks/markdown.ts

@@ -183,6 +183,7 @@ export function getMarkdown(prNumber: number) {
   const benchmark = JSON.parse(fs.readFileSync(inputFile, 'utf-8'));
   const baseBenchmark = getBaseBenchmark();
 
+  const metricsByThreads = Metrics.filter(m => m.groupBy === 'threads').map(m => m.name);
   const metricsByBlockSize = Metrics.filter(m => m.groupBy === 'block-size').map(m => m.name);
   const metricsByChainLength = Metrics.filter(m => m.groupBy === 'chain-length').map(m => m.name);
   const kernelCircuitMetrics = Metrics.filter(m => m.groupBy === 'protocol-circuit-name').map(m => m.name);
@@ -218,6 +219,11 @@ All benchmarks are run on txs on the \`Benchmarking\` contract on the repository
 ${prSourceDataText}
 ${baseCommitText}
 
+### Proof generation
+
+Each column represents the number of threads used in proof generation.
+${getTableContent(pick(benchmark, metricsByThreads), baseBenchmark, 'threads')}
+
 ### L2 block published to L1
 
 Each column represents the number of txs on an L2 block published to L1.