Merge branch 'master' into gd/noir_issue_4330

* master: (92 commits)
  feat(acir)!: Add predicate to call opcode (#5616)
  feat: only export values from accumulated data (#5604)
  feat: Improve the proving orchestration lifecycle (#5535)
  chore(docs): Random updates (#5281)
  feat: `add` and `sub` methods of `EasyPrivateUint` throw when called in public (#5581)
  Update quickstart.md (#5605)
  chore(docs): Fix indexed-merkle-tree docs images (#4674)
  feat: Sync from noir (#5572)
  feat: /foundry is canoncial build of foundry. e2e tests can start own anvil. (#5522)
  fix: remove EFS lifecycle rule (#5587)
  fix: invalid fork terraform (#5585)
  fix(ci): Install fixed foundry version in CI (#5582)
  docs(spec): hashing and keys (#5478)
  fix: mainnet fork redeploys (#5573)
  git subrepo push --branch=master noir-projects/aztec-nr
  git_subrepo.sh: Fix parent in .gitrepo file. [skip ci]
  chore: replace relative paths to noir-protocol-circuits
  git subrepo push --branch=master barretenberg
  refactor: nuking accounts from e2e setup (#5574)
  feat(avm): Track gas from memory accesses explicitly (#5563)
  ...

.circleci/config.yml

@@ -1182,6 +1182,11 @@ jobs:
             deploy_dockerhub cli
             deploy_dockerhub aztec-faucet
             deploy_dockerhub mainnet-fork
+      - run:
+          name: "Deploy mainnet fork"
+          command: |
+            should_deploy || exit 0
+            deploy_terraform_services iac/mainnet-fork mainnet-fork mainnet-fork aws_efs_file_system.aztec_mainnet_fork_data_store
       - run:
           name: "Release canary to NPM: bb.js"
           command: |
@@ -1212,11 +1217,6 @@ jobs:
           command: |
             should_release || exit 0
             deploy_npm l1-contracts latest
-      - run:
-          name: "Deploy mainnet fork"
-          command: |
-            should_deploy || exit 0
-            deploy_terraform_services iac/mainnet-fork
       - run:
           name: "Deploy L1 contracts to mainnet fork"
           working_directory: l1-contracts
@@ -1244,7 +1244,7 @@ jobs:
             # Check if l1-contracts have changed
             if [ "$CONTRACTS_DEPLOYED" -eq 1 ]; then
               echo "Contracts have changed, taint nodes to force redeploy.."
-              deploy_terraform_services yarn-project/aztec-node aztec aztec-node "aws_ecs_task_definition.aztec-node[0],aws_ecs_task_definition.aztec-node[1]"
+              deploy_terraform_services yarn-project/aztec-node aztec aztec-node "aws_ecs_task_definition.aztec-node[0],aws_ecs_task_definition.aztec-node[1]" 1
             else
               deploy_terraform_services yarn-project/aztec-node aztec
             fi
@@ -1334,6 +1334,7 @@ workflows:
           <<: *defaults
       - noir-packages-tests:
           requires:
+            - bb-js
             - noir-ecr-manifest
             - noir-packages
           <<: *defaults
@@ -1556,6 +1557,7 @@ workflows:
             - guides-sample-dapp
             - guides-up-quick-start
             - yellow-paper
+            - noir-packages-tests
             - yarn-project-test
           <<: *defaults
 

.github/ci-setup-action/action.yml

@@ -0,0 +1,35 @@
+# Reusable setup workflow for CI tasks
+name: Setup Workflow
+description: 'Reusable setup steps'
+
+inputs:
+  dockerhub_password:
+    required: true
+    description: 'DockerHub Password'
+  github_actor:
+    required: true
+    description: 'GitHub Actor'
+
+runs:
+  # define an action, runs in OS of caller
+  using: composite
+  steps:
+    - name: Cache Submodules
+      id: cache-submodules
+      uses: actions/cache@v3
+      with:
+        path: .git/modules
+        key: submodules-${{ hashFiles('.gitmodules') }}
+
+    - name: Checkout Submodules
+      shell: bash
+      run: git submodule sync --recursive && git submodule update --init --recursive
+
+    - name: Setup Earthly
+      uses: earthly/actions-setup@v1
+      with:
+        version: 'v0.8.5'
+
+    - name: Setup Env
+      shell: bash
+      run: ./scripts/setup_env.sh ${{ inputs.dockerhub_password }} ${{ inputs.github_actor }}

.github/workflows/ci.yml

@@ -9,69 +9,74 @@ jobs:
   # there's a lot of x86 tasks - let's split out the build step
   build-x86:
     runs-on: ubuntu-latest
+    outputs:
+      e2e_list: ${{ steps.e2e_list.outputs.list }}
     env:
       EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }}
     # cancel if reran on same PR if exists, otherwise if on same commit
     concurrency:
-      group: build-${{ github.event.pull_request.number || github.ref_name }}-x86
-      cancel-in-progress: ${{ github.ref_name != 'master' }}
+      group: build-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86
+      cancel-in-progress: true
     steps:
-      - uses: earthly/actions-setup@v1
-        with:
-          version: v0.8.5
-
       - name: Checkout
         uses: actions/checkout@v4
         with:
+          # we check out submodules in ci-setup-action
           ref: ${{ github.event.pull_request.head.sha }}
-          submodules: recursive
 
       - name: Setup
-        working-directory: ./scripts
-        run: ./setup_env.sh ${{ secrets.DOCKERHUB_PASSWORD }} ${{ github.actor }}
+        uses: ./.github/ci-setup-action
+        with:
+          dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }}
+          github_actor: ${{ github.actor }} 
 
-      - name: Test
+      - name: Build
         working-directory: ./yarn-project
-        if: github.actor == 'ludamad' || github.actor == 'charlielye' || github.actor == 'sklppy88' || github.actor == 'alexghr' || github.actor == 'benesjan' || github.actor == 'spalladino' || github.actor == 'just-mitch'
         run: |
-          # push to registry
           earthly-cloud build x86 --push +build-end-to-end
 
+      # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end
+      # (Note ARM uses just 2 tests as a smoketest)
+      - id: e2e_list
+        working-directory: ./yarn-project/end-to-end
+        run: |
+          echo "list=$(earthly ls | grep -v '+base' |  sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT
+
   # all the end-to-end integration tests for aztec
   e2e-arm:
     runs-on: ubuntu-latest
     env:
       EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }}
     strategy:
+      fail-fast: false
       matrix:
         test:
           - e2e-card-game
           - e2e-crowdfunding-and-claim
     # cancel if reran on same PR if exists, otherwise if on same commit
     concurrency:
-      group: ${{ matrix.test }}-${{ github.event.pull_request.number || github.ref_name }}-arm
-      cancel-in-progress: ${{ github.ref_name != 'master' }}
+      group: ${{ matrix.test }}-${{ github.ref_name == 'master' && github.run_id|| github.ref_name }}-arm
+      cancel-in-progress: true
     steps:
-      - uses: earthly/actions-setup@v1
-        with:
-          version: v0.8.5
-
       - name: Checkout
         uses: actions/checkout@v4
         with:
+          # we check out submodules in ci-setup-action
           ref: ${{ github.event.pull_request.head.sha }}
-          submodules: recursive
 
       - name: Setup
-        working-directory: ./scripts
-        run: ./setup_env.sh ${{ secrets.DOCKERHUB_PASSWORD }} ${{ github.actor }}
+        uses: ./.github/ci-setup-action
+        with:
+          dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }}
+          github_actor: ${{ github.actor }}
 
-      - name: Test
+      - name: Build
         working-directory: ./yarn-project/end-to-end/
-        if: github.actor == 'ludamad' || github.actor == 'charlielye' || github.actor == 'sklppy88' || github.actor == 'alexghr' || github.actor == 'benesjan' || github.actor == 'spalladino' || github.actor == 'just-mitch'
         # We don't do much on arm, just run it on their builder
-        run: 
-          earthly-cloud build arm --no-output +${{ matrix.test }} --e2e_remote=true
+        run:
+          # Flags:
+          # - e2e_mode=build to get images from a local build, not a dockerhub registry strategy
+          earthly-cloud build arm --no-output +${{ matrix.test }} --e2e_mode=build
 
   # all the end-to-end integration tests for aztec
   e2e-x86:
@@ -80,100 +85,72 @@ jobs:
     env:
       EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }}
     strategy:
+      fail-fast: false
       matrix:
-        test:
-          - e2e-state-vars
-          - e2e-block-building
-          - e2e-nested-contract
-          - e2e-static-calls
-          - e2e-delegate-calls
-          - e2e-non-contract-account
-          - e2e-cross-chain-messaging
-          - e2e-crowdfunding-and-claim
-          - e2e-public-cross-chain-messaging
-          - e2e-public-to-private-messaging
-          - e2e-account-contracts
-          - e2e-escrow-contract
-          - e2e-inclusion-proofs-contract
-          - e2e-pending-note-hashes-contract
-          - e2e-ordering
-          - uniswap-trade-on-l1-from-l2
-          - integration-archiver-l1-to-l2
-          - integration-l1-publisher
-          - e2e-cli
-          - e2e-persistence
-          # - e2e-browser # TODO failing
-          - e2e-card-game
-          - e2e-avm-simulator
-          - e2e-fees
-          - e2e-dapp-subscription
-          - pxe
-          - cli-docs-sandbox
-          # - e2e-docs-examples # TODO failing
+        test: ${{ fromJson( needs.build-x86.outputs.e2e_list )}}
     # cancel if reran on same PR if exists, otherwise if on same commit
     concurrency:
-      group: ${{ matrix.test }}-${{ github.event.pull_request.number || github.ref_name }}-x86
-      cancel-in-progress: ${{ github.ref_name != 'master' }}
+      group: ${{ matrix.test }}-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86
+      cancel-in-progress: true
     steps:
-      - uses: earthly/actions-setup@v1
-        with:
-          version: v0.8.5
-
       - name: Checkout
         uses: actions/checkout@v4
         with:
+          # we check out submodules in ci-setup-action
           ref: ${{ github.event.pull_request.head.sha }}
-          submodules: recursive
 
       - name: Setup
-        working-directory: ./scripts
-        run: ./setup_env.sh ${{ secrets.DOCKERHUB_PASSWORD }} ${{ github.actor }}
+        uses: ./.github/ci-setup-action
+        with:
+          dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }}
+          github_actor: ${{ github.actor }} 
 
       - name: Pull E2E Images
         working-directory: ./barretenberg/cpp/
-        if: github.actor == 'ludamad' || github.actor == 'charlielye' || github.actor == 'sklppy88' || github.actor == 'alexghr' || github.actor == 'benesjan' || github.actor == 'spalladino' || github.actor == 'just-mitch'
         run: |
           docker pull aztecprotocol/aztec-cache:$(git rev-parse HEAD)
           docker pull aztecprotocol/end-to-end-cache:$(git rev-parse HEAD)
+          docker pull aztecprotocol/foundry-nightly-de33b6af53005037b463318d2628b5cfcaf3991-amd64:latest
           docker tag aztecprotocol/aztec-cache:$(git rev-parse HEAD) aztecprotocol/aztec:latest
           docker tag aztecprotocol/end-to-end-cache:$(git rev-parse HEAD) aztecprotocol/end-to-end:latest
-
+          docker tag aztecprotocol/foundry-nightly-de33b6af53005037b463318d2628b5cfcaf3991-amd64:latest aztecprotocol/foundry-nightly-de33b6af53005037b463318d2628b5cfcaf3991-amd64:latest
       - name: Test
         working-directory: ./yarn-project/end-to-end/
-        if: github.actor == 'ludamad' || github.actor == 'charlielye' || github.actor == 'sklppy88' || github.actor == 'alexghr' || github.actor == 'benesjan' || github.actor == 'spalladino' || github.actor == 'just-mitch'
         run: |
-          earthly-cloud build x86 --no-output +${{ matrix.test }}
+          earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache
 
   # barretenberg (prover) native tests
   bb-native-tests:
     runs-on: ubuntu-latest
     env:
       EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }}
-    strategy: { matrix: { environment: [x86, arm] } }
+    strategy:
+      fail-fast: false
+      matrix:
+        environment: [x86]
+        # pending fix for intermittent test
+        # environment: [x86, arm]
     # cancel if reran on same PR if exists, otherwise if on same commit
     concurrency:
-      group: bb-native-tests-${{ github.event.pull_request.number || github.ref_name }}-${{ matrix.environment }}
-      cancel-in-progress: ${{ github.ref_name != 'master' }}
+      group: bb-native-tests-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-${{ matrix.environment }}
+      cancel-in-progress: true
     steps:
-      - uses: earthly/actions-setup@v1
-        with:
-          version: v0.8.5
-
       - name: Checkout
         uses: actions/checkout@v4
         with:
+          # we check out submodules in ci-setup-action
           ref: ${{ github.event.pull_request.head.sha }}
-          submodules: recursive
 
       - name: Setup
-        working-directory: ./scripts
-        run: ./setup_env.sh ${{ secrets.DOCKERHUB_PASSWORD }} ${{ github.actor }}
+        uses: ./.github/ci-setup-action
+        with:
+          dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }}
+          github_actor: ${{ github.actor }} 
 
       - name: Test
         working-directory: ./barretenberg/cpp/
-        if: github.actor == 'ludamad' || github.actor == 'charlielye' || github.actor == 'sklppy88' || github.actor == 'alexghr' || github.actor == 'benesjan' || github.actor == 'spalladino' || github.actor == 'just-mitch'
         run: |
-          earthly-cloud build ${{ matrix.environment }} --no-output +test
+          earthly-cloud test ${{ matrix.environment }} --no-output +test
 
   # All benchmarks, purposefully ran sequential on a machine
   # they should use parallelism within the benchmark, but only one thing should run at a time
@@ -185,30 +162,40 @@ jobs:
       EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }}
     # cancel if reran on same PR if exists, otherwise if on same commit
     concurrency:
-      # TODO no longer exclusive.. revisit this
-      group: ${{ github.actor }}-x86
-      cancel-in-progress: ${{ github.ref_name != 'master' }}
+      # We use a looser concurrency limit here, but we place a lock 
+      # in the actual step action so that only one bench takes place
+      group: bench-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86
+      cancel-in-progress: true
     steps:
-      - uses: earthly/actions-setup@v1
-        with:
-          version: v0.8.5
-
       - name: Checkout
         uses: actions/checkout@v4
         with:
+          # we check out submodules in ci-setup-action
           ref: ${{ github.event.pull_request.head.sha }}
-          submodules: recursive
 
       - name: Setup
-        working-directory: ./scripts
-        run: ./setup_env.sh ${{ secrets.DOCKERHUB_PASSWORD }} ${{ github.actor }}
+        uses: ./.github/ci-setup-action
+        with:
+          dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }}
+          github_actor: ${{ github.actor }} 
+
+      - name: Build and Push Binaries
+        working-directory: ./barretenberg/cpp/
+        run: earthly-cloud build x86 --push +bench-base
+
+      # Only allow one person to use this runner
+      # The reason is that as detailed in https://github.com/ben-z/gh-action-mutex
+      # things do not become 'pending' in github actions, and instead just cancel one another
+      - name: Set up mutex
+        uses: ben-z/gh-action-mutex@v1.0.0-alpha.9
+        with:
+          branch: gh-action-mutex-bench
 
+      # Use bench_mode=cache to read the pushed build above
       - name: Client IVC Bench
         working-directory: ./barretenberg/cpp/
-        if: github.actor == 'ludamad' || github.actor == 'charlielye' || github.actor == 'sklppy88' || github.actor == 'alexghr' || github.actor == 'benesjan' || github.actor == 'spalladino' || github.actor == 'just-mitch'
-        run: earthly-cloud bench x86 --no-output +bench-client-ivc
+        run: earthly-cloud bench x86 --no-output +bench-client-ivc --bench_mode=cache
 
       - name: Ultrahonk Bench
         working-directory: ./barretenberg/cpp/
-        if: github.actor == 'ludamad' || github.actor == 'charlielye' || github.actor == 'sklppy88' || github.actor == 'alexghr' || github.actor == 'benesjan' || github.actor == 'spalladino' || github.actor == 'just-mitch'
-        run: earthly-cloud bench x86 --no-output +bench-ultra-honk
+        run: earthly-cloud bench x86 --no-output +bench-ultra-honk --bench_mode=cache

.github/workflows/protocol-circuits-gate-diff.yml

@@ -8,10 +8,9 @@ on:
 
 jobs:
   compare_protocol_circuits_gates:
-    # cancel if reran on same PR if exists, otherwise if on same commit
     concurrency:
-      group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-      cancel-in-progress: ${{ github.ref_name != 'master' }}
+      group: compare_protocol_circuits_gates-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}
+      cancel-in-progress: true
     if: "!startsWith(github.head_ref, 'release-please--')"
     runs-on: ubuntu-20.04
     steps: