From 41984b4e43fd3fd42522552ecb8ca1e54f32cdf1 Mon Sep 17 00:00:00 2001
From: ludamad <adam.domurad@gmail.com>
Date: Fri, 23 Feb 2024 21:32:44 +0000
Subject: [PATCH] feat(devops): filter circleci config no-ops (#4731)

This is more efficient than letting a bunch of no-op machines provision
---
 .circleci/config.yml                          | 113 ++++++++++++++++-
 .../scripts/generate_circleci_config.py       | 117 ++++++++++++++++++
 build_manifest.yml                            |  10 +-
 3 files changed, 232 insertions(+), 8 deletions(-)
 create mode 100755 build-system/scripts/generate_circleci_config.py

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 87ae7654998..eb276b39bdf 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -16,8 +16,10 @@
 
 version: 2.1
 
+setup: true # have a dynamic config step
 orbs:
-  slack: circleci/slack@4.12.1
+  continuation: circleci/continuation@1.0.0
+  slack: circleci/slack@4.12.5
 
 parameters:
   workflow:
@@ -69,6 +71,21 @@ setup_env: &setup_env
     command: ./build-system/scripts/setup_env "$CIRCLE_SHA1" "$CIRCLE_TAG" "$CIRCLE_JOB" "$CIRCLE_REPOSITORY_URL" "$CIRCLE_BRANCH" "$CIRCLE_PULL_REQUEST"
 
 jobs:
+  # Dynamically filter our code, quickly figuring out which jobs we can skip.
+  generate-config:
+    docker:
+      - image: aztecprotocol/alpine-build-image
+    resource_class: xlarge
+    steps:
+      - *checkout
+      - *setup_env
+      - run:
+          name: Generate Pipeline generated_config.yml file
+          command: |
+            # filter our circleci config to the minimal pipeline
+            build-system/scripts/generate_circleci_config.py > .circleci/generated_config.yml
+      - continuation/continue:
+          configuration_path: .circleci/generated_config.yml
   # Noir
   noir-x86_64:
     docker:
@@ -80,6 +97,7 @@ jobs:
       - run:
           name: "Build"
           command: cond_spot_run_build noir 32
+          aztec_manifest_key: noir
 
   noir-arm64:
     docker:
@@ -91,6 +109,7 @@ jobs:
       - run:
           name: "Build"
           command: cond_spot_run_build noir 32 arm64
+          aztec_manifest_key: noir
 
   noir-ecr-manifest:
     machine:
@@ -102,6 +121,7 @@ jobs:
       - run:
           name: "Create ECR manifest"
           command: create_ecr_manifest noir x86_64,arm64
+          aztec_manifest_key: noir
 
   noir-packages:
     docker:
@@ -113,6 +133,7 @@ jobs:
       - run:
           name: "Build"
           command: cond_spot_run_build noir-packages 32
+          aztec_manifest_key: noir-packages
 
   noir-compile-acir-tests:
     docker:
@@ -124,6 +145,7 @@ jobs:
       - run:
           name: "Build"
           command: cond_spot_run_build noir-compile-acir-tests 32
+          aztec_manifest_key: noir
 
   avm-transpiler:
     docker:
@@ -135,6 +157,7 @@ jobs:
       - run:
           name: "Build"
           command: cond_spot_run_build avm-transpiler 32
+          aztec_manifest_key: avm-transpiler
 
   # Barretenberg
   barretenberg-wasm-linux-clang:
@@ -147,6 +170,7 @@ jobs:
       - run:
           name: "Build"
           command: cond_spot_run_build barretenberg-wasm-linux-clang 128
+          aztec_manifest_key: barretenberg-wasm-linux-clang
 
   barretenberg-x86_64-linux-gcc:
     docker:
@@ -158,6 +182,7 @@ jobs:
       - run:
           name: "Build"
           command: cond_spot_run_build barretenberg-x86_64-linux-gcc 128
+          aztec_manifest_key: barretenberg-x86_64-linux-gcc
 
   barretenberg-x86_64-linux-clang:
     docker:
@@ -169,6 +194,7 @@ jobs:
       - run:
           name: "Build"
           command: cond_spot_run_build barretenberg-x86_64-linux-clang 128
+          aztec_manifest_key: barretenberg-x86_64-linux-clang
 
   barretenberg-x86_64-linux-clang-fuzzing:
     docker:
@@ -180,7 +206,8 @@ jobs:
       - run:
           name: "Build"
           command: cond_spot_run_build barretenberg-x86_64-linux-clang-fuzzing 128
-
+          aztec_manifest_key: barretenberg-x86_64-linux-clang-fuzzing
+          
   barretenberg-x86_64-linux-clang-assert:
     docker:
       - image: aztecprotocol/alpine-build-image
@@ -191,6 +218,7 @@ jobs:
       - run:
           name: "Build"
           command: cond_spot_run_build barretenberg-x86_64-linux-clang-assert 128
+          aztec_manifest_key: barretenberg-x86_64-linux-clang-assert
 
   barretenberg-x86_64-linux-clang-sol:
     docker:
@@ -202,6 +230,7 @@ jobs:
       - run:
           name: "Build"
           command: cond_spot_run_build barretenberg-x86_64-linux-clang-sol 32
+          aztec_manifest_key: barretenberg-x86_64-linux-clang-sol
 
   barretenberg-docs:
     machine:
@@ -213,6 +242,7 @@ jobs:
       - run:
           name: "Build barretenberg docs"
           command: build barretenberg-docs
+          aztec_manifest_key: barretenberg-docs
       - run:
           name: "Deploy barretenberg docs"
           command: |
@@ -228,6 +258,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_test barretenberg-x86_64-linux-clang-assert 32 ./scripts/run_tests 1 stdlib-tests
+          aztec_manifest_key: barretenberg-x86_64-linux-clang-assert
 
   barretenberg-dsl-tests:
     docker:
@@ -239,6 +270,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_test barretenberg-x86_64-linux-clang-assert 32 ./scripts/run_tests 1 dsl_tests
+          aztec_manifest_key: barretenberg-x86_64-linux-clang-assert
 
   barretenberg-tests:
     docker:
@@ -250,6 +282,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_test barretenberg-x86_64-linux-clang-assert 32 ./scripts/bb-tests.sh
+          aztec_manifest_key: barretenberg-x86_64-linux-clang-assert
 
   barretenberg-bench:
     machine:
@@ -263,6 +296,7 @@ jobs:
       - run:
           name: "Benchmark"
           command: cond_spot_run_build barretenberg-bench 32
+          aztec_manifest_key: barretenberg-bench
       - run:
           name: "Upload"
           command: |
@@ -278,6 +312,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_test barretenberg-x86_64-linux-clang-assert 32 ./scripts/run_tests 1 proof_system_tests
+          aztec_manifest_key: barretenberg-x86_64-linux-clang-assert
 
   barretenberg-stdlib-recursion-ultra-tests:
     docker:
@@ -289,6 +324,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_test barretenberg-x86_64-linux-clang-assert 32 ./scripts/run_tests 3 stdlib_recursion_tests --gtest_filter=-*turbo*
+          aztec_manifest_key: barretenberg-x86_64-linux-clang-assert
 
   barretenberg-join-split-tests:
     docker:
@@ -300,6 +336,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_test barretenberg-x86_64-linux-clang-assert 32 ./scripts/run_tests 3 join_split_example_proofs_join_split_tests --gtest_filter=-*full_proof*
+          aztec_manifest_key: barretenberg-x86_64-linux-clang-assert
 
   barretenberg-acir-tests-bb:
     docker:
@@ -311,6 +348,7 @@ jobs:
       - run:
           name: "Build and test"
           command: cond_spot_run_build barretenberg-acir-tests-bb 32
+          aztec_manifest_key: barretenberg-acir-tests-bb
 
   bb-js:
     machine:
@@ -322,6 +360,7 @@ jobs:
       - run:
           name: "Build and test"
           command: build bb.js
+          aztec_manifest_key: bb.js
 
   bb-js-tests:
     docker:
@@ -333,6 +372,7 @@ jobs:
       - run:
           name: "Build and test"
           command: cond_spot_run_test bb.js 32 ./scripts/run_tests
+          aztec_manifest_key: bb.js
 
   bb-js-acir-tests:
     docker:
@@ -344,6 +384,7 @@ jobs:
       - run:
           name: "Build and test"
           command: cond_spot_run_build barretenberg-acir-tests-bb.js 32
+          aztec_manifest_key: barretenberg-acir-tests-bb.js
 
   l1-contracts:
     machine:
@@ -355,6 +396,7 @@ jobs:
       - run:
           name: "Build and test"
           command: build l1-contracts
+          aztec_manifest_key: l1-contracts
 
   noir-projects:
     machine:
@@ -366,6 +408,7 @@ jobs:
       - run:
           name: "Build and test"
           command: build noir-projects
+          aztec_manifest_key: noir-projects
 
   boxes-files:
     machine:
@@ -377,6 +420,7 @@ jobs:
       - run:
           name: "Build"
           command: build boxes-files
+          aztec_manifest_key: boxes-files
 
   yarn-project-base:
     machine:
@@ -388,6 +432,7 @@ jobs:
       - run:
           name: "Build"
           command: build yarn-project-base | add_timestamps
+          aztec_manifest_key: yarn-project-base
 
   yarn-project:
     machine:
@@ -399,6 +444,7 @@ jobs:
       - run:
           name: Build
           command: build yarn-project | add_timestamps
+          aztec_manifest_key: yarn-project
 
   yarn-project-prod:
     machine:
@@ -410,6 +456,7 @@ jobs:
       - run:
           name: Build
           command: build yarn-project-prod | add_timestamps
+          aztec_manifest_key: yarn-project-prod
 
   yarn-project-formatting:
     docker:
@@ -421,6 +468,7 @@ jobs:
       - run:
           name: Check Formatting
           command: cond_spot_run_container yarn-project 8 formatting | add_timestamps
+          aztec_manifest_key: yarn-project
 
   yarn-project-tests:
     docker:
@@ -432,6 +480,7 @@ jobs:
       - run:
           name: Test
           command: cond_spot_run_container yarn-project 64 test | add_timestamps
+          aztec_manifest_key: yarn-project
 
   aztec-package:
     machine:
@@ -443,6 +492,7 @@ jobs:
       - run:
           name: "Build and test"
           command: build aztec
+          aztec_manifest_key: aztec
 
   cli:
     machine:
@@ -454,6 +504,7 @@ jobs:
       - run:
           name: "Build and test"
           command: build cli
+          aztec_manifest_key: cli
 
   mainnet-fork:
     machine:
@@ -465,6 +516,7 @@ jobs:
       - run:
           name: "Build"
           command: build mainnet-fork | add_timestamps
+          aztec_manifest_key: mainnet-fork
 
   aztec-faucet:
     machine:
@@ -476,6 +528,7 @@ jobs:
       - run:
           name: "Build and test"
           command: build aztec-faucet | add_timestamps
+          aztec_manifest_key: aztec-faucet
 
   boxes:
     docker:
@@ -487,6 +540,7 @@ jobs:
       - run:
           name: "Build"
           command: cond_spot_run_build boxes 4
+          aztec_manifest_key: boxes
 
   boxes-vanilla:
     docker:
@@ -498,6 +552,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose boxes 4 ./docker-compose.yml BOX=box-vanilla
+          aztec_manifest_key: boxes
 
   boxes-react:
     docker:
@@ -509,6 +564,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose boxes 4 ./docker-compose.yml BOX=box-react
+          aztec_manifest_key: boxes
 
   end-to-end:
     machine:
@@ -520,6 +576,7 @@ jobs:
       - run:
           name: "Build"
           command: build end-to-end
+          aztec_manifest_key: end-to-end
 
   e2e-2-pxes:
     docker:
@@ -531,6 +588,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_2_pxes.test.ts
+          aztec_manifest_key: end-to-end
 
   e2e-note-getter:
     docker:
@@ -542,6 +600,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_note_getter.test.ts
+          aztec_manifest_key: end-to-end
 
   e2e-counter:
     docker:
@@ -553,6 +612,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_counter_contract.test.ts
+          aztec_manifest_key: end-to-end
 
   e2e-private-voting:
     docker:
@@ -564,6 +624,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_private_voting_contract.test.ts
+          aztec_manifest_key: end-to-end
 
   e2e-multiple-accounts-1-enc-key:
     docker:
@@ -575,6 +636,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_multiple_accounts_1_enc_key.test.ts
+          aztec_manifest_key: end-to-end
 
   e2e-deploy-contract:
     docker:
@@ -586,6 +648,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_deploy_contract.test.ts
+          aztec_manifest_key: end-to-end
 
   e2e-lending-contract:
     docker:
@@ -597,6 +660,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_lending_contract.test.ts
+          aztec_manifest_key: end-to-end
 
   e2e-token-contract:
     docker:
@@ -608,6 +672,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_token_contract.test.ts
+          aztec_manifest_key: end-to-end
 
   e2e-blacklist-token-contract:
     docker:
@@ -619,6 +684,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_blacklist_token_contract.test.ts
+          aztec_manifest_key: end-to-end
 
   # TODO(3458): Investigate intermittent failure
   # e2e-slow-tree:
@@ -631,6 +697,7 @@ jobs:
   #     - run:
   #         name: "Test"
   #         command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_slow_tree.test.ts
+  #         aztec_manifest_key: end-to-end
 
   e2e-sandbox-example:
     docker:
@@ -642,6 +709,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_sandbox_example.test.ts
+          aztec_manifest_key: end-to-end
 
   e2e-state-vars:
     docker:
@@ -653,6 +721,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_state_vars.test.ts
+          aztec_manifest_key: end-to-end
 
   e2e-block-building:
     docker:
@@ -664,6 +733,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_block_building.test.ts
+          aztec_manifest_key: end-to-end
 
   e2e-nested-contract:
     docker:
@@ -675,6 +745,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_nested_contract.test.ts
+          aztec_manifest_key: end-to-end
 
   e2e-static-calls:
     docker:
@@ -686,6 +757,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_static_calls.test.ts
+          aztec_manifest_key: end-to-end
 
   e2e-delegate-calls:
     docker:
@@ -697,6 +769,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_delegate_calls.test.ts
+          aztec_manifest_key: end-to-end
 
   e2e-non-contract-account:
     docker:
@@ -708,6 +781,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_non_contract_account.test.ts
+          aztec_manifest_key: end-to-end
 
   e2e-cross-chain-messaging:
     docker:
@@ -719,6 +793,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_cross_chain_messaging.test.ts
+          aztec_manifest_key: end-to-end
 
   e2e-public-cross-chain-messaging:
     docker:
@@ -730,6 +805,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_public_cross_chain_messaging.test.ts
+          aztec_manifest_key: end-to-end
 
   e2e-public-to-private-messaging:
     docker:
@@ -741,6 +817,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_public_to_private_messaging.test.ts
+          aztec_manifest_key: end-to-end
 
   e2e-account-contracts:
     docker:
@@ -752,6 +829,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_account_contracts.test.ts
+          aztec_manifest_key: end-to-end
 
   e2e-escrow-contract:
     docker:
@@ -763,6 +841,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_escrow_contract.test.ts
+          aztec_manifest_key: end-to-end
 
   e2e-inclusion-proofs-contract:
     docker:
@@ -774,6 +853,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_inclusion_proofs_contract.test.ts
+          aztec_manifest_key: end-to-end
 
   e2e-pending-commitments-contract:
     docker:
@@ -785,6 +865,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_pending_commitments_contract.test.ts
+          aztec_manifest_key: end-to-end
 
   e2e-ordering:
     docker:
@@ -796,6 +877,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_ordering.test.ts
+          aztec_manifest_key: end-to-end
 
   uniswap-trade-on-l1-from-l2:
     docker:
@@ -807,6 +889,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=uniswap_trade_on_l1_from_l2.test.ts
+          aztec_manifest_key: end-to-end
 
   integration-archiver-l1-to-l2:
     docker:
@@ -818,6 +901,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=integration_archiver_l1_to_l2.test.ts
+          aztec_manifest_key: end-to-end
 
   integration-l1-publisher:
     docker:
@@ -829,6 +913,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=integration_l1_publisher.test.ts
+          aztec_manifest_key: end-to-end
 
   e2e-cli:
     docker:
@@ -840,6 +925,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_cli.test.ts
+          aztec_manifest_key: end-to-end
 
   e2e-persistence:
     docker:
@@ -851,6 +937,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=e2e_persistence.test.ts
+          aztec_manifest_key: end-to-end
 
   e2e-browser:
     docker:
@@ -862,6 +949,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-browser.yml TEST=e2e_aztec_js_browser.test.ts
+          aztec_manifest_key: end-to-end
 
   e2e-card-game:
     docker:
@@ -873,6 +961,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_card_game.test.ts
+          aztec_manifest_key: end-to-end
 
   e2e-avm-simulator:
     docker:
@@ -884,6 +973,7 @@ jobs:
       - run:
           name: "Test"
           command: AVM_ENABLED=1 cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_avm_simulator.test.ts
+          aztec_manifest_key: end-to-end
 
   e2e-fees:
     docker:
@@ -895,6 +985,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_fees.test.ts
+          aztec_manifest_key: end-to-end
 
   pxe:
     docker:
@@ -906,6 +997,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=pxe_sandbox.test.ts
+          aztec_manifest_key: end-to-end
 
   cli-docs-sandbox:
     docker:
@@ -917,6 +1009,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=cli_docs_sandbox.test.ts
+          aztec_manifest_key: end-to-end
 
   e2e-docs-examples:
     docker:
@@ -928,6 +1021,7 @@ jobs:
       - run:
           name: "Test"
           command: AVM_ENABLED=1 cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=docs_examples_test.ts
+          aztec_manifest_key: end-to-end
 
   guides-writing-an-account-contract:
     docker:
@@ -939,6 +1033,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=guides/writing_an_account_contract.test.ts
+          aztec_manifest_key: end-to-end
 
   guides-dapp-testing:
     docker:
@@ -950,6 +1045,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=guides/dapp_testing.test.ts
+          aztec_manifest_key: end-to-end
 
   guides-sample-dapp:
     docker:
@@ -961,6 +1057,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=sample-dapp
+          aztec_manifest_key: end-to-end
 
   guides-up-quick-start:
     docker:
@@ -972,6 +1069,7 @@ jobs:
       - run:
           name: "Test"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=guides/up_quick_start.test.ts
+          aztec_manifest_key: end-to-end
 
   bench-publish-rollup:
     docker:
@@ -983,6 +1081,7 @@ jobs:
       - run:
           name: "Benchmark"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=benchmarks/bench_publish_rollup.test.ts DEBUG=aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees
+          aztec_manifest_key: end-to-end
 
   bench-process-history:
     docker:
@@ -994,6 +1093,7 @@ jobs:
       - run:
           name: "Benchmark"
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=benchmarks/bench_process_history.test.ts DEBUG=aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees
+          aztec_manifest_key: end-to-end
 
   build-docs:
     machine:
@@ -1019,6 +1119,7 @@ jobs:
       - run:
           name: "Build docs"
           command: build docs
+          aztec_manifest_key: docs
       - run:
           name: "Deploy docs"
           command: |
@@ -1039,6 +1140,7 @@ jobs:
       - run:
           name: "Build yellow paper"
           command: build yellow-paper
+          aztec_manifest_key: yellow-paper
 
   e2e-join:
     docker:
@@ -1207,9 +1309,14 @@ bb_test: &bb_test
 
 # Workflows.
 workflows:
+  setup-workflow:
+    jobs:
+      - generate-config
   system:
     when:
-      equal: [system, << pipeline.parameters.workflow >>]
+      # Used to generate a dynamic 'system' workflow
+      # This is rewritten to 'system' on the real workflow (otherwise this is ignored by circleci)
+      equal: [NEVER, << pipeline.parameters.workflow >>]
     jobs:
       # Noir
       - noir-x86_64: *defaults
diff --git a/build-system/scripts/generate_circleci_config.py b/build-system/scripts/generate_circleci_config.py
new file mode 100755
index 00000000000..f04328646d0
--- /dev/null
+++ b/build-system/scripts/generate_circleci_config.py
@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+# Operates on circleci (loaded as json) from stdin
+# Outputs filtered circleci without the jobs we don't need to run
+# NOTE: This uses the build manifest YAML file to filter the dependency graph in CircleCI BUT it is not one-to-one.
+# There is a heuristic here where we expect a job to be associated with a manifest job if it lists the build_manifest.yml job name in its command with a known build command.
+import json
+import yaml
+import re
+from concurrent.futures import ProcessPoolExecutor, as_completed
+import subprocess
+import sys
+
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
+# same functionality as query_manifest rebuildPatterns but in bulk
+def get_manifest_job_names():
+    manifest = yaml.safe_load(open("build_manifest.yml"))
+    return list(manifest)
+
+def is_already_built_circleci_job(circleci_job, already_built_manifest_jobs):
+    """
+    This function checks if a given CircleCI job is associated with a specific already-built manifest job. 
+    It does so by checking the job's steps for an 'aztec_manifest_key' that contain references to manifest names.
+    We want to see at least one such key, and for all such keys to be in 'already_built_manifest_jobs'.
+    """
+    steps = circleci_job.get("steps", [])
+    matching_steps = 0
+    for step in steps:
+        run_info = step.get("run", "")
+        # Check if run_info is a string, short-hand notation
+        if isinstance(run_info, str):
+            # if there's no run key, or we use string short-hand, continue
+            continue
+        keys = run_info.get("aztec_manifest_key", [])
+        if isinstance(keys, str):
+            keys = [keys]
+        if not keys: # empty list? continue
+            continue
+        for key in keys:
+            if key not in already_built_manifest_jobs:
+                # We have found a different string here - bail out
+                return False
+        matching_steps += 1
+    # All steps have matched - but make sure that's actually more than one step
+    return matching_steps > 0 
+
+def get_already_built_circleci_job_names(circleci_jobs):
+    already_built_manifest_jobs = list(get_already_built_manifest_job_names())
+    for key in already_built_manifest_jobs:
+        eprint("Detected cached manifest key:", key)
+    for job_name, circleci_job in circleci_jobs.items():
+        if is_already_built_circleci_job(circleci_job, already_built_manifest_jobs):
+            yield job_name
+
+# Helper for multiprocessing
+def _get_already_built_manifest_job_names(manifest_name):
+    content_hash = subprocess.check_output(['calculate_content_hash', manifest_name]).decode("utf-8")
+    completed = subprocess.run(["check_rebuild", f"cache-{content_hash}", manifest_name], stdout=subprocess.DEVNULL)
+    if completed.returncode == 0:
+        return manifest_name
+    else:
+        return None
+
+def get_already_built_manifest_job_names():
+    manifest_names = get_manifest_job_names()
+
+    with ProcessPoolExecutor() as executor:
+        futures = {executor.submit(_get_already_built_manifest_job_names, key): key for key in manifest_names}
+        for future in as_completed(futures):
+            result = future.result()
+            if result is not None:
+                yield result
+
+def remove_jobs_from_workflow(jobs, to_remove):
+    """
+    Removes jobs from a given CircleCI JSON workflow.
+    
+    Parameters:
+        jobs (dict): The JSON object representing the CircleCI workflow jobs dependencies portion.
+        to_remove (list): The list of jobs to be removed from the workflow.
+    
+    Returns:
+        dict: The new JSON object with specified jobs removed.
+    """
+
+    new_jobs = []
+    # Remove specified jobs
+    for job in jobs:
+        key = next(iter(job))
+        if key in to_remove:
+            continue
+        # remove our filtered jobs from the dependency graph via the requires attribute
+        job[key]["requires"] = [r for r in job[key].get("requires", []) if r not in jobs_to_remove]
+        new_jobs.append(job)
+    return new_jobs
+
+if __name__ == '__main__':
+    # The CircleCI workflow as a JSON string (Replace this with your actual workflow)
+    
+    # Convert the JSON string to a Python dictionary
+    workflow_dict = yaml.safe_load(open('.circleci/config.yml'))
+
+    # # List of jobs to remove
+    jobs_to_remove = list(get_already_built_circleci_job_names(workflow_dict["jobs"]))
+    for key in jobs_to_remove:
+        eprint("Skipping circleci job:", key)
+
+    # Get rid of workflow setup step and setup flag
+    workflow_dict["setup"] = False
+    del workflow_dict["workflows"]["setup-workflow"]
+    # Remove the jobs and get the new workflow
+    workflow_dict["workflows"]["system"]["jobs"] = remove_jobs_from_workflow(workflow_dict["workflows"]["system"]["jobs"], jobs_to_remove)
+    workflow_dict["workflows"]["system"]["when"] = {"equal":["system","<< pipeline.parameters.workflow >>"]}
+    # Convert the new workflow back to JSON string
+    new_workflow_json_str = json.dumps(workflow_dict, indent=2)
+    print(new_workflow_json_str)
\ No newline at end of file
diff --git a/build_manifest.yml b/build_manifest.yml
index bac4b7bc86e..665e16f6b24 100644
--- a/build_manifest.yml
+++ b/build_manifest.yml
@@ -207,11 +207,11 @@ docs:
   dockerfile: docs/Dockerfile
   rebuildPatterns:
     - ^docs/
-    - ^.*.cpp$
-    - ^.*.hpp$
-    - ^.*.ts$
-    - ^.release-please-manifest.json$
-    - ^.*.nr$
+    - ^.*\.cpp$
+    - ^.*\.hpp$
+    - ^.*\.ts$
+    - ^.release-please-manifest\.json$
+    - ^.*\.nr$
   dependencies:
     - yarn-project
     - noir-projects