From d115e1312331eb59aa70396203548c1bf559ca28 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 10:26:34 +0000 Subject: [PATCH 01/47] initial circleci config filter --- .circleci/config.yml | 26 +++++- .../scripts/generate_circleci_config.py | 90 +++++++++++++++++++ 2 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 build-system/scripts/generate_circleci_config.py diff --git a/.circleci/config.yml b/.circleci/config.yml index a1cec4bf45e..31d197633b8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,8 +16,10 @@ version: 2.1 +setup: true # have a dynamic config step orbs: - slack: circleci/slack@4.12.1 + continuation: circleci/continuation@1.0.0 + slack: circleci/slack@4.12.5 parameters: workflow: @@ -69,6 +71,21 @@ setup_env: &setup_env command: ./build-system/scripts/setup_env "$CIRCLE_SHA1" "$CIRCLE_TAG" "$CIRCLE_JOB" "$CIRCLE_REPOSITORY_URL" "$CIRCLE_BRANCH" "$CIRCLE_PULL_REQUEST" jobs: + # Dynamically filter our code, quickly figuring out which jobs we can skip. + generate-config: + docker: + - image: aztecprotocol/alpine-build-image + steps: + - *checkout + - *setup_env + - run: + name: Generate Pipeline generated_config.yml file + command: | + # convert to json, run a python script, write out json (which is a readable as YAML) + pip install PyYAML + build-system/scripts/generate_circleci_config.py > .circleci/generated_config.yml + - continuation/continue: + configuration_path: .circleci/generated_config.yml # Noir noir-x86_64: docker: @@ -1207,9 +1224,14 @@ bb_test: &bb_test # Workflows. workflows: + setup-workflow: + jobs: + - generate-config system: when: - equal: [system, << pipeline.parameters.workflow >>] + # Used to generate a dynamic 'system' workflow + # This is rewritten to 'system' on the real workflow (otherwise this is ignored by circleci) + equal: [NEVER, << pipeline.parameters.workflow >>] jobs: # Noir - noir-x86_64: *defaults diff --git a/build-system/scripts/generate_circleci_config.py b/build-system/scripts/generate_circleci_config.py new file mode 100644 index 00000000000..76937414077 --- /dev/null +++ b/build-system/scripts/generate_circleci_config.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +# Operates on circleci (loaded as json) from stdin +# Outputs filtered circleci without the jobs we don't need to run +import json +import yaml +from concurrent.futures import ProcessPoolExecutor, as_completed +import subprocess + +# same functionality as query_manifest rebuildPatterns but in bulk +def get_manifest_job_names(): + manifest = json.load(open("build_manifest.json")) + return list(manifest) + +def has_associated_manifest_job(circleci_job, manifest_names): + steps = circleci_job.get("steps", []) + for step in steps: + run_info = step.get("run", {}) + command = run_info.get("command", "") + for manifest_name in manifest_names: + if manifest_name in command: + return True + return False + + +def get_already_built_circleci_job_names(circleci_jobs): + manifest_names = list(get_already_built_manifest_job_names()) + for job_name, circleci_job in circleci_jobs.items(): + if has_associated_manifest_job(circleci_job, manifest_names): + yield job_name + +# Helper for multiprocessing +def _get_already_built_manifest_job_names(manifest_name): + content_hash = subprocess.check_output(['calculate_content_hash', manifest_name]).decode("utf-8") + completed = subprocess.run(["check_rebuild", f"cache-{content_hash}", manifest_name], stdout=subprocess.DEVNULL) + if completed.returncode == 0: + return manifest_name + else: + return None + +def get_already_built_manifest_job_names(): + manifest_names = get_manifest_job_names() + + with ProcessPoolExecutor() as executor: + futures = {executor.submit(_get_already_built_manifest_job_names, key): key for key in manifest_names} + for future in as_completed(futures): + result = future.result() + if result is not None: + yield result + +def remove_jobs_from_workflow(jobs, to_remove): + """ + Removes jobs from a given CircleCI JSON workflow. + + Parameters: + jobs (dict): The JSON object representing the CircleCI workflow jobs dependencies portion. + to_remove (list): The list of jobs to be removed from the workflow. + + Returns: + dict: The new JSON object with specified jobs removed. + """ + + new_jobs = [] + # Remove specified jobs + for job in jobs: + key = next(iter(job)) + if key in to_remove: + continue + # remove our filtered jobs from the dependency graph via the requires attribute + job[key]["requires"] = [r for r in job[key].get("requires", []) if r not in jobs_to_remove] + new_jobs.append(job) + return new_jobs + +if __name__ == '__main__': + # The CircleCI workflow as a JSON string (Replace this with your actual workflow) + + # Convert the JSON string to a Python dictionary + workflow_dict = yaml.safe_load(open('.circleci/config.yml')) + + # # List of jobs to remove + jobs_to_remove = list(get_already_built_circleci_job_names(workflow_dict["jobs"])) + + # Get rid of workflow setup step and setup flag + workflow_dict["setup"] = False + del workflow_dict["workflows"]["setup-workflow"] + # Remove the jobs and get the new workflow + workflow_dict["workflows"]["system"]["jobs"] = remove_jobs_from_workflow(workflow_dict["workflows"]["system"]["jobs"], jobs_to_remove) + workflow_dict["workflows"]["system"]["when"] = {"equal":["system","<< pipeline.parameters.workflow >>"]} + # Convert the new workflow back to JSON string + new_workflow_json_str = json.dumps(workflow_dict, indent=2) + print(new_workflow_json_str) \ No newline at end of file From 9f6b7f198cb00f647bd3c258acd7e9fb15936494 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 10:38:51 +0000 Subject: [PATCH 02/47] jiggle content hash --- build-system/scripts/generate_circleci_config.py | 6 +++--- noir/README.md | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/build-system/scripts/generate_circleci_config.py b/build-system/scripts/generate_circleci_config.py index 76937414077..3e46177fb29 100644 --- a/build-system/scripts/generate_circleci_config.py +++ b/build-system/scripts/generate_circleci_config.py @@ -8,7 +8,7 @@ # same functionality as query_manifest rebuildPatterns but in bulk def get_manifest_job_names(): - manifest = json.load(open("build_manifest.json")) + manifest = yaml.load(open("build_manifest.yml")) return list(manifest) def has_associated_manifest_job(circleci_job, manifest_names): @@ -21,7 +21,6 @@ def has_associated_manifest_job(circleci_job, manifest_names): return True return False - def get_already_built_circleci_job_names(circleci_jobs): manifest_names = list(get_already_built_manifest_job_names()) for job_name, circleci_job in circleci_jobs.items(): @@ -64,6 +63,7 @@ def remove_jobs_from_workflow(jobs, to_remove): for job in jobs: key = next(iter(job)) if key in to_remove: + print(key) continue # remove our filtered jobs from the dependency graph via the requires attribute job[key]["requires"] = [r for r in job[key].get("requires", []) if r not in jobs_to_remove] @@ -87,4 +87,4 @@ def remove_jobs_from_workflow(jobs, to_remove): workflow_dict["workflows"]["system"]["when"] = {"equal":["system","<< pipeline.parameters.workflow >>"]} # Convert the new workflow back to JSON string new_workflow_json_str = json.dumps(workflow_dict, indent=2) - print(new_workflow_json_str) \ No newline at end of file + # print(new_workflow_json_str) \ No newline at end of file diff --git a/noir/README.md b/noir/README.md index 771c3f1c74d..f63afa88ada 100644 --- a/noir/README.md +++ b/noir/README.md @@ -2,6 +2,7 @@ Noir is a Domain Specific Language for SNARK proving systems. It has been designed to use any ACIR compatible proving system. + **This implementation is in early development. It has not been reviewed or audited. It is not suitable to be used in production. Expect bugs!** ## Quick Start From 824272503a285b0c30e85c7b672eacfc16216a9f Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 10:39:54 +0000 Subject: [PATCH 03/47] jiggle content hash --- noir/acvm-repo/acvm/src/compiler/optimizers/general.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/noir/acvm-repo/acvm/src/compiler/optimizers/general.rs b/noir/acvm-repo/acvm/src/compiler/optimizers/general.rs index 2bd781f7bb5..0165ef03c4c 100644 --- a/noir/acvm-repo/acvm/src/compiler/optimizers/general.rs +++ b/noir/acvm-repo/acvm/src/compiler/optimizers/general.rs @@ -17,6 +17,7 @@ impl GeneralOptimizer { } } + // Remove all terms with zero as a coefficient fn remove_zero_coefficients(mut opcode: Expression) -> Expression { // Check the mul terms From c9d4bc8eb73e426e1288353e1fd5b7919cc19c08 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 10:40:16 +0000 Subject: [PATCH 04/47] jiggle content hash --- noir/acvm-repo/acvm/src/compiler/optimizers/general.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/noir/acvm-repo/acvm/src/compiler/optimizers/general.rs b/noir/acvm-repo/acvm/src/compiler/optimizers/general.rs index 0165ef03c4c..2bd781f7bb5 100644 --- a/noir/acvm-repo/acvm/src/compiler/optimizers/general.rs +++ b/noir/acvm-repo/acvm/src/compiler/optimizers/general.rs @@ -17,7 +17,6 @@ impl GeneralOptimizer { } } - // Remove all terms with zero as a coefficient fn remove_zero_coefficients(mut opcode: Expression) -> Expression { // Check the mul terms From 1acbecf10eaf464d3e785c04e2790608ffbcbc62 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 10:41:39 +0000 Subject: [PATCH 05/47] unjiggle. ship it --- build-system/scripts/generate_circleci_config.py | 2 +- noir/README.md | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/build-system/scripts/generate_circleci_config.py b/build-system/scripts/generate_circleci_config.py index 3e46177fb29..b86d1874b64 100644 --- a/build-system/scripts/generate_circleci_config.py +++ b/build-system/scripts/generate_circleci_config.py @@ -87,4 +87,4 @@ def remove_jobs_from_workflow(jobs, to_remove): workflow_dict["workflows"]["system"]["when"] = {"equal":["system","<< pipeline.parameters.workflow >>"]} # Convert the new workflow back to JSON string new_workflow_json_str = json.dumps(workflow_dict, indent=2) - # print(new_workflow_json_str) \ No newline at end of file + print(new_workflow_json_str) \ No newline at end of file diff --git a/noir/README.md b/noir/README.md index f63afa88ada..771c3f1c74d 100644 --- a/noir/README.md +++ b/noir/README.md @@ -2,7 +2,6 @@ Noir is a Domain Specific Language for SNARK proving systems. It has been designed to use any ACIR compatible proving system. - **This implementation is in early development. It has not been reviewed or audited. It is not suitable to be used in production. Expect bugs!** ## Quick Start From 9095bf2f5b49a3fb91c925dcdbe080621080499e Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 10:43:14 +0000 Subject: [PATCH 06/47] give exec perms to generate script --- build-system/scripts/generate_circleci_config.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 build-system/scripts/generate_circleci_config.py diff --git a/build-system/scripts/generate_circleci_config.py b/build-system/scripts/generate_circleci_config.py old mode 100644 new mode 100755 From eb9139d4272fee8dde0c208857e3281abd027cca Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 10:45:57 +0000 Subject: [PATCH 07/47] fix script --- build-system/scripts/generate_circleci_config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build-system/scripts/generate_circleci_config.py b/build-system/scripts/generate_circleci_config.py index b86d1874b64..81733d0493e 100755 --- a/build-system/scripts/generate_circleci_config.py +++ b/build-system/scripts/generate_circleci_config.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 # Operates on circleci (loaded as json) from stdin # Outputs filtered circleci without the jobs we don't need to run +# NOTE: This uses the build manifest YAML file to filter the dependency graph in CircleCI BUT it is not one-to-one. +# There is a heuristic here where we expect a job to be associated with a manifest job if it lists the build_manifest.yml job name in its command. import json import yaml from concurrent.futures import ProcessPoolExecutor, as_completed @@ -8,7 +10,7 @@ # same functionality as query_manifest rebuildPatterns but in bulk def get_manifest_job_names(): - manifest = yaml.load(open("build_manifest.yml")) + manifest = yaml.safe_load(open("build_manifest.yml")) return list(manifest) def has_associated_manifest_job(circleci_job, manifest_names): From 1faaa15b4c50e15a458ac704fba4987131bc238b Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 10:50:59 +0000 Subject: [PATCH 08/47] fix script --- build-system/scripts/generate_circleci_config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build-system/scripts/generate_circleci_config.py b/build-system/scripts/generate_circleci_config.py index 81733d0493e..d99506e0e0d 100755 --- a/build-system/scripts/generate_circleci_config.py +++ b/build-system/scripts/generate_circleci_config.py @@ -80,6 +80,7 @@ def remove_jobs_from_workflow(jobs, to_remove): # # List of jobs to remove jobs_to_remove = list(get_already_built_circleci_job_names(workflow_dict["jobs"])) + print(jobs_to_remove) # Get rid of workflow setup step and setup flag workflow_dict["setup"] = False @@ -89,4 +90,4 @@ def remove_jobs_from_workflow(jobs, to_remove): workflow_dict["workflows"]["system"]["when"] = {"equal":["system","<< pipeline.parameters.workflow >>"]} # Convert the new workflow back to JSON string new_workflow_json_str = json.dumps(workflow_dict, indent=2) - print(new_workflow_json_str) \ No newline at end of file + # print(new_workflow_json_str) \ No newline at end of file From 4a8a19bf5c941465c6b894fb32a7401e4470333a Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 10:54:20 +0000 Subject: [PATCH 09/47] fix script --- build-system/scripts/generate_circleci_config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build-system/scripts/generate_circleci_config.py b/build-system/scripts/generate_circleci_config.py index d99506e0e0d..83d3ca7bcfd 100755 --- a/build-system/scripts/generate_circleci_config.py +++ b/build-system/scripts/generate_circleci_config.py @@ -65,7 +65,6 @@ def remove_jobs_from_workflow(jobs, to_remove): for job in jobs: key = next(iter(job)) if key in to_remove: - print(key) continue # remove our filtered jobs from the dependency graph via the requires attribute job[key]["requires"] = [r for r in job[key].get("requires", []) if r not in jobs_to_remove] @@ -80,7 +79,6 @@ def remove_jobs_from_workflow(jobs, to_remove): # # List of jobs to remove jobs_to_remove = list(get_already_built_circleci_job_names(workflow_dict["jobs"])) - print(jobs_to_remove) # Get rid of workflow setup step and setup flag workflow_dict["setup"] = False @@ -90,4 +88,6 @@ def remove_jobs_from_workflow(jobs, to_remove): workflow_dict["workflows"]["system"]["when"] = {"equal":["system","<< pipeline.parameters.workflow >>"]} # Convert the new workflow back to JSON string new_workflow_json_str = json.dumps(workflow_dict, indent=2) - # print(new_workflow_json_str) \ No newline at end of file + # for t in workflow_dict["workflows"]["system"]["jobs"]: + # print(t) + print(new_workflow_json_str) \ No newline at end of file From 1c2f9c441f7a277f6719b57cd25fc66c8a94029a Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 10:56:18 +0000 Subject: [PATCH 10/47] introspect --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 31d197633b8..a2480ab8a1e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -84,6 +84,8 @@ jobs: # convert to json, run a python script, write out json (which is a readable as YAML) pip install PyYAML build-system/scripts/generate_circleci_config.py > .circleci/generated_config.yml + cat .circleci/generated_config.yml + echo "[]" > .circleci/generated_config.yml - continuation/continue: configuration_path: .circleci/generated_config.yml # Noir From 7f04598d1ad1c0450c992ae3bbb6daa2792a5108 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 11:05:03 +0000 Subject: [PATCH 11/47] inspect --- build-system/scripts/generate_circleci_config.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build-system/scripts/generate_circleci_config.py b/build-system/scripts/generate_circleci_config.py index 83d3ca7bcfd..61a28861658 100755 --- a/build-system/scripts/generate_circleci_config.py +++ b/build-system/scripts/generate_circleci_config.py @@ -32,6 +32,7 @@ def get_already_built_circleci_job_names(circleci_jobs): # Helper for multiprocessing def _get_already_built_manifest_job_names(manifest_name): content_hash = subprocess.check_output(['calculate_content_hash', manifest_name]).decode("utf-8") + eprint(manifest_name, content_hash) completed = subprocess.run(["check_rebuild", f"cache-{content_hash}", manifest_name], stdout=subprocess.DEVNULL) if completed.returncode == 0: return manifest_name @@ -70,6 +71,10 @@ def remove_jobs_from_workflow(jobs, to_remove): job[key]["requires"] = [r for r in job[key].get("requires", []) if r not in jobs_to_remove] new_jobs.append(job) return new_jobs +import sys + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) if __name__ == '__main__': # The CircleCI workflow as a JSON string (Replace this with your actual workflow) From fab158239dc8e513f5149f4966140e6a92c374ab Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 11:11:00 +0000 Subject: [PATCH 12/47] inspect --- build-system/scripts/generate_circleci_config.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/build-system/scripts/generate_circleci_config.py b/build-system/scripts/generate_circleci_config.py index 61a28861658..8c001622ede 100755 --- a/build-system/scripts/generate_circleci_config.py +++ b/build-system/scripts/generate_circleci_config.py @@ -32,7 +32,6 @@ def get_already_built_circleci_job_names(circleci_jobs): # Helper for multiprocessing def _get_already_built_manifest_job_names(manifest_name): content_hash = subprocess.check_output(['calculate_content_hash', manifest_name]).decode("utf-8") - eprint(manifest_name, content_hash) completed = subprocess.run(["check_rebuild", f"cache-{content_hash}", manifest_name], stdout=subprocess.DEVNULL) if completed.returncode == 0: return manifest_name @@ -84,6 +83,9 @@ def eprint(*args, **kwargs): # # List of jobs to remove jobs_to_remove = list(get_already_built_circleci_job_names(workflow_dict["jobs"])) + for remove in jobs_to_remove: + eprint("REMOVE") + eprint(remove) # Get rid of workflow setup step and setup flag workflow_dict["setup"] = False @@ -93,6 +95,7 @@ def eprint(*args, **kwargs): workflow_dict["workflows"]["system"]["when"] = {"equal":["system","<< pipeline.parameters.workflow >>"]} # Convert the new workflow back to JSON string new_workflow_json_str = json.dumps(workflow_dict, indent=2) - # for t in workflow_dict["workflows"]["system"]["jobs"]: - # print(t) - print(new_workflow_json_str) \ No newline at end of file + for t in workflow_dict["workflows"]["system"]["jobs"]: + eprint("KEPT") + eprint(t) + # print(new_workflow_json_str) \ No newline at end of file From 5b51c6af9a077b794fe6badea4360a76e100d218 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 11:13:11 +0000 Subject: [PATCH 13/47] ok --- build-system/scripts/generate_circleci_config.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/build-system/scripts/generate_circleci_config.py b/build-system/scripts/generate_circleci_config.py index 8c001622ede..c1f0d791271 100755 --- a/build-system/scripts/generate_circleci_config.py +++ b/build-system/scripts/generate_circleci_config.py @@ -25,6 +25,8 @@ def has_associated_manifest_job(circleci_job, manifest_names): def get_already_built_circleci_job_names(circleci_jobs): manifest_names = list(get_already_built_manifest_job_names()) + for man in manifest_names: + eprint("MAN", man) for job_name, circleci_job in circleci_jobs.items(): if has_associated_manifest_job(circleci_job, manifest_names): yield job_name @@ -83,10 +85,6 @@ def eprint(*args, **kwargs): # # List of jobs to remove jobs_to_remove = list(get_already_built_circleci_job_names(workflow_dict["jobs"])) - for remove in jobs_to_remove: - eprint("REMOVE") - eprint(remove) - # Get rid of workflow setup step and setup flag workflow_dict["setup"] = False del workflow_dict["workflows"]["setup-workflow"] From 9028dad2ff4c2de9f4ac3f3454b1e92980f2a1ea Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 11:16:11 +0000 Subject: [PATCH 14/47] inspect-it --- build-system/scripts/generate_circleci_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build-system/scripts/generate_circleci_config.py b/build-system/scripts/generate_circleci_config.py index c1f0d791271..6fef0f881f5 100755 --- a/build-system/scripts/generate_circleci_config.py +++ b/build-system/scripts/generate_circleci_config.py @@ -48,6 +48,7 @@ def get_already_built_manifest_job_names(): for future in as_completed(futures): result = future.result() if result is not None: + eprint("res", result) yield result def remove_jobs_from_workflow(jobs, to_remove): From 78984358dce98ab06213fb2f4db97d6c98e59c63 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 11:18:50 +0000 Subject: [PATCH 15/47] Update --- build-system/scripts/generate_circleci_config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build-system/scripts/generate_circleci_config.py b/build-system/scripts/generate_circleci_config.py index 6fef0f881f5..2a95e6bdaed 100755 --- a/build-system/scripts/generate_circleci_config.py +++ b/build-system/scripts/generate_circleci_config.py @@ -44,11 +44,12 @@ def get_already_built_manifest_job_names(): manifest_names = get_manifest_job_names() with ProcessPoolExecutor() as executor: - futures = {executor.submit(_get_already_built_manifest_job_names, key): key for key in manifest_names} + futures = [executor.submit(_get_already_built_manifest_job_names, key) for key in manifest_names] for future in as_completed(futures): + eprint('fut', future) result = future.result() + eprint('res', result) if result is not None: - eprint("res", result) yield result def remove_jobs_from_workflow(jobs, to_remove): From 5a8648f408a3777c563d8c878761e7fa231d2cdb Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 11:38:58 +0000 Subject: [PATCH 16/47] basic test --- .../scripts/generate_circleci_config.py | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/build-system/scripts/generate_circleci_config.py b/build-system/scripts/generate_circleci_config.py index 2a95e6bdaed..050402b609e 100755 --- a/build-system/scripts/generate_circleci_config.py +++ b/build-system/scripts/generate_circleci_config.py @@ -43,14 +43,19 @@ def _get_already_built_manifest_job_names(manifest_name): def get_already_built_manifest_job_names(): manifest_names = get_manifest_job_names() - with ProcessPoolExecutor() as executor: - futures = [executor.submit(_get_already_built_manifest_job_names, key) for key in manifest_names] - for future in as_completed(futures): - eprint('fut', future) - result = future.result() - eprint('res', result) - if result is not None: - yield result + for key in manifest_names: + res = _get_already_built_manifest_job_names(key) + if res is not None: + yield res + + # with ProcessPoolExecutor() as executor: + # futures = [executor.submit(_get_already_built_manifest_job_names, key) for key in manifest_names] + # for future in as_completed(futures): + # eprint('fut', future) + # result = future.result() + # eprint('res', result) + # if result is not None: + # yield result def remove_jobs_from_workflow(jobs, to_remove): """ From 1dbdea038e8e45285f4c1a5ecf71166a8158db87 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 11:45:10 +0000 Subject: [PATCH 17/47] [ci debug] --- build-system/scripts/generate_circleci_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build-system/scripts/generate_circleci_config.py b/build-system/scripts/generate_circleci_config.py index 050402b609e..bb63f9c9ff5 100755 --- a/build-system/scripts/generate_circleci_config.py +++ b/build-system/scripts/generate_circleci_config.py @@ -34,6 +34,7 @@ def get_already_built_circleci_job_names(circleci_jobs): # Helper for multiprocessing def _get_already_built_manifest_job_names(manifest_name): content_hash = subprocess.check_output(['calculate_content_hash', manifest_name]).decode("utf-8") + eprint("CON", content_hash) completed = subprocess.run(["check_rebuild", f"cache-{content_hash}", manifest_name], stdout=subprocess.DEVNULL) if completed.returncode == 0: return manifest_name From 732bda3ce5bdfc7b1b16a85cc2632b868efc3a6e Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 11:50:49 +0000 Subject: [PATCH 18/47] ecr_login needed! ci debug helped --- .circleci/config.yml | 2 ++ .../scripts/generate_circleci_config.py | 28 ++++++------------- build-system/scripts/image_exists | 1 + 3 files changed, 12 insertions(+), 19 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a2480ab8a1e..4e8c94f3108 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -81,6 +81,8 @@ jobs: - run: name: Generate Pipeline generated_config.yml file command: | + # ability to query ECR images + ecr_login # convert to json, run a python script, write out json (which is a readable as YAML) pip install PyYAML build-system/scripts/generate_circleci_config.py > .circleci/generated_config.yml diff --git a/build-system/scripts/generate_circleci_config.py b/build-system/scripts/generate_circleci_config.py index bb63f9c9ff5..9c6713380dc 100755 --- a/build-system/scripts/generate_circleci_config.py +++ b/build-system/scripts/generate_circleci_config.py @@ -25,8 +25,6 @@ def has_associated_manifest_job(circleci_job, manifest_names): def get_already_built_circleci_job_names(circleci_jobs): manifest_names = list(get_already_built_manifest_job_names()) - for man in manifest_names: - eprint("MAN", man) for job_name, circleci_job in circleci_jobs.items(): if has_associated_manifest_job(circleci_job, manifest_names): yield job_name @@ -34,7 +32,6 @@ def get_already_built_circleci_job_names(circleci_jobs): # Helper for multiprocessing def _get_already_built_manifest_job_names(manifest_name): content_hash = subprocess.check_output(['calculate_content_hash', manifest_name]).decode("utf-8") - eprint("CON", content_hash) completed = subprocess.run(["check_rebuild", f"cache-{content_hash}", manifest_name], stdout=subprocess.DEVNULL) if completed.returncode == 0: return manifest_name @@ -44,19 +41,12 @@ def _get_already_built_manifest_job_names(manifest_name): def get_already_built_manifest_job_names(): manifest_names = get_manifest_job_names() - for key in manifest_names: - res = _get_already_built_manifest_job_names(key) - if res is not None: - yield res - - # with ProcessPoolExecutor() as executor: - # futures = [executor.submit(_get_already_built_manifest_job_names, key) for key in manifest_names] - # for future in as_completed(futures): - # eprint('fut', future) - # result = future.result() - # eprint('res', result) - # if result is not None: - # yield result + with ProcessPoolExecutor() as executor: + futures = {executor.submit(_get_already_built_manifest_job_names, key): key for key in manifest_names} + for future in as_completed(futures): + result = future.result() + if result is not None: + yield result def remove_jobs_from_workflow(jobs, to_remove): """ @@ -93,6 +83,7 @@ def eprint(*args, **kwargs): # # List of jobs to remove jobs_to_remove = list(get_already_built_circleci_job_names(workflow_dict["jobs"])) + # Get rid of workflow setup step and setup flag workflow_dict["setup"] = False del workflow_dict["workflows"]["setup-workflow"] @@ -102,6 +93,5 @@ def eprint(*args, **kwargs): # Convert the new workflow back to JSON string new_workflow_json_str = json.dumps(workflow_dict, indent=2) for t in workflow_dict["workflows"]["system"]["jobs"]: - eprint("KEPT") - eprint(t) - # print(new_workflow_json_str) \ No newline at end of file + eprint("KEPT", t) + print(new_workflow_json_str) \ No newline at end of file diff --git a/build-system/scripts/image_exists b/build-system/scripts/image_exists index 0ad9d90ae69..6e142d7cfb2 100755 --- a/build-system/scripts/image_exists +++ b/build-system/scripts/image_exists @@ -1,4 +1,5 @@ #!/usr/bin/env bash +[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu # Returns true if the given image exists in the current ECR. aws ecr describe-images --region=$ECR_REGION --repository-name=$1 --image-ids=imageTag=$2 > /dev/null 2>&1 From eab1e9a5c2cfce8dab63abe2e4dd1f85dd7fd878 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 11:53:35 +0000 Subject: [PATCH 19/47] compatible image?? --- .circleci/config.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4e8c94f3108..8fb33a8f790 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -73,8 +73,9 @@ setup_env: &setup_env jobs: # Dynamically filter our code, quickly figuring out which jobs we can skip. generate-config: - docker: - - image: aztecprotocol/alpine-build-image + machine: + image: ubuntu-2204:2023.07.2 + resource_class: large steps: - *checkout - *setup_env From b9ac9524afc12bccfcb23b0b7f665e6b1bcfbb68 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 11:58:05 +0000 Subject: [PATCH 20/47] [ci debug] From c60c6835c2c15895e60fba613f3ba98209bca418 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 12:36:26 +0000 Subject: [PATCH 21/47] token --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8fb33a8f790..c096f2eaa47 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -82,6 +82,7 @@ jobs: - run: name: Generate Pipeline generated_config.yml file command: | + echo "$AZTEC_GITHUB_TOKEN" # ability to query ECR images ecr_login # convert to json, run a python script, write out json (which is a readable as YAML) From 399a19193297f22592a78d06b4c4419b75b2f5c9 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 12:38:08 +0000 Subject: [PATCH 22/47] token --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c096f2eaa47..3af7f051ebc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -82,7 +82,7 @@ jobs: - run: name: Generate Pipeline generated_config.yml file command: | - echo "$AZTEC_GITHUB_TOKEN" + echo "$AWS_SECRET_ACCESS_KEY" # ability to query ECR images ecr_login # convert to json, run a python script, write out json (which is a readable as YAML) From 888cc1bda5f97af486adc7083f1da1cc5232071f Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 14:03:06 +0000 Subject: [PATCH 23/47] . --- build-system/scripts/generate_circleci_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-system/scripts/generate_circleci_config.py b/build-system/scripts/generate_circleci_config.py index 9c6713380dc..75effecd99a 100755 --- a/build-system/scripts/generate_circleci_config.py +++ b/build-system/scripts/generate_circleci_config.py @@ -2,7 +2,7 @@ # Operates on circleci (loaded as json) from stdin # Outputs filtered circleci without the jobs we don't need to run # NOTE: This uses the build manifest YAML file to filter the dependency graph in CircleCI BUT it is not one-to-one. -# There is a heuristic here where we expect a job to be associated with a manifest job if it lists the build_manifest.yml job name in its command. +# There is a heuristic here where we expect a job to be associated with a manifest job if it lists the build_manifest.yml job name in its command with a known build command. import json import yaml from concurrent.futures import ProcessPoolExecutor, as_completed From d8640c3c2f277c7bb796f05d4c2d5ead141fb0aa Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 14:28:01 +0000 Subject: [PATCH 24/47] heuristic --- .../scripts/generate_circleci_config.py | 44 +++++++++++++++---- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/build-system/scripts/generate_circleci_config.py b/build-system/scripts/generate_circleci_config.py index 75effecd99a..8b3da289668 100755 --- a/build-system/scripts/generate_circleci_config.py +++ b/build-system/scripts/generate_circleci_config.py @@ -5,28 +5,52 @@ # There is a heuristic here where we expect a job to be associated with a manifest job if it lists the build_manifest.yml job name in its command with a known build command. import json import yaml +import re from concurrent.futures import ProcessPoolExecutor, as_completed import subprocess +# Define the regex pattern with known prefixes in uppercase +KNOWN_PREFIXES_PATTERN = r'(BUILD|COND_SPOT_RUN_BUILD|COND_SPOT_RUN_TEST|COND_SPOT_RUN_CONTAINER|COND_SPOT_RUN_COMPOSE)[\s_-]+(\w[\w_-]*)' + # same functionality as query_manifest rebuildPatterns but in bulk def get_manifest_job_names(): manifest = yaml.safe_load(open("build_manifest.yml")) return list(manifest) -def has_associated_manifest_job(circleci_job, manifest_names): +def is_already_built_circleci_job(circleci_job, already_built_manifest_jobs): + """ + This function checks if a given CircleCI job is associated with a specific already-built manifest job. + It does so by analyzing the job's steps for commands that contain references to manifest names. + The association is based on a heuristic where a job is considered linked to a manifest job if + its command includes the name of the manifest. This is not a direct one-to-one mapping but + rather a pattern-matching approach using known command prefixes and the build manifest YAML file. + The method ensures that only one matching step exists in the CircleCI job that aligns with this criteria. + """ steps = circleci_job.get("steps", []) + matching_steps = 0 for step in steps: run_info = step.get("run", {}) - command = run_info.get("command", "") - for manifest_name in manifest_names: - if manifest_name in command: - return True - return False + # Check if run_info is a string, short-hand notation + if isinstance(run_info, str): + command = run_info + else: + command = run_info.get("command", "") + match = re.search(KNOWN_PREFIXES_PATTERN, command, re.IGNORECASE) + if not match: + continue + matched_manifest = match.group(2) + if matched_manifest in already_built_manifest_jobs: + matching_steps += 1 + else: + # We have found a different string here - bail out + return False + # All steps have matched - but make sure that's actually more than one step + return matching_steps > 0 def get_already_built_circleci_job_names(circleci_jobs): - manifest_names = list(get_already_built_manifest_job_names()) + already_built_manifest_jobs = list(get_already_built_manifest_job_names()) for job_name, circleci_job in circleci_jobs.items(): - if has_associated_manifest_job(circleci_job, manifest_names): + if is_already_built_circleci_job(circleci_job, already_built_manifest_jobs): yield job_name # Helper for multiprocessing @@ -83,6 +107,8 @@ def eprint(*args, **kwargs): # # List of jobs to remove jobs_to_remove = list(get_already_built_circleci_job_names(workflow_dict["jobs"])) + for key in jobs_to_remove: + eprint("KEY", key) # Get rid of workflow setup step and setup flag workflow_dict["setup"] = False @@ -94,4 +120,4 @@ def eprint(*args, **kwargs): new_workflow_json_str = json.dumps(workflow_dict, indent=2) for t in workflow_dict["workflows"]["system"]["jobs"]: eprint("KEPT", t) - print(new_workflow_json_str) \ No newline at end of file + # print(new_workflow_json_str) \ No newline at end of file From 4ced47966f44ca22ca39479566e53528eb69b5d7 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 14:44:17 +0000 Subject: [PATCH 25/47] robustness via aztec_manifest_key --- .circleci/config.yml | 4 +++ .../scripts/generate_circleci_config.py | 34 ++++++++----------- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3af7f051ebc..053fee95ab7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -103,6 +103,7 @@ jobs: - run: name: "Build" command: cond_spot_run_build noir 32 + aztec_manifest_key: noir # for circleci build filtering noir-arm64: docker: @@ -114,6 +115,7 @@ jobs: - run: name: "Build" command: cond_spot_run_build noir 32 arm64 + aztec_manifest_key: noir # for circleci build filtering noir-ecr-manifest: machine: @@ -125,6 +127,7 @@ jobs: - run: name: "Create ECR manifest" command: create_ecr_manifest noir x86_64,arm64 + aztec_manifest_key: noir # for circleci build filtering noir-packages: docker: @@ -136,6 +139,7 @@ jobs: - run: name: "Build" command: cond_spot_run_build noir-packages 32 + aztec_manifest_key: noir-packages # for circleci build filtering noir-compile-acir-tests: docker: diff --git a/build-system/scripts/generate_circleci_config.py b/build-system/scripts/generate_circleci_config.py index 8b3da289668..7bac7495566 100755 --- a/build-system/scripts/generate_circleci_config.py +++ b/build-system/scripts/generate_circleci_config.py @@ -9,9 +9,6 @@ from concurrent.futures import ProcessPoolExecutor, as_completed import subprocess -# Define the regex pattern with known prefixes in uppercase -KNOWN_PREFIXES_PATTERN = r'(BUILD|COND_SPOT_RUN_BUILD|COND_SPOT_RUN_TEST|COND_SPOT_RUN_CONTAINER|COND_SPOT_RUN_COMPOSE)[\s_-]+(\w[\w_-]*)' - # same functionality as query_manifest rebuildPatterns but in bulk def get_manifest_job_names(): manifest = yaml.safe_load(open("build_manifest.yml")) @@ -20,30 +17,27 @@ def get_manifest_job_names(): def is_already_built_circleci_job(circleci_job, already_built_manifest_jobs): """ This function checks if a given CircleCI job is associated with a specific already-built manifest job. - It does so by analyzing the job's steps for commands that contain references to manifest names. - The association is based on a heuristic where a job is considered linked to a manifest job if - its command includes the name of the manifest. This is not a direct one-to-one mapping but - rather a pattern-matching approach using known command prefixes and the build manifest YAML file. - The method ensures that only one matching step exists in the CircleCI job that aligns with this criteria. + It does so by checking the job's steps for an 'aztec_manifest_key' that contain references to manifest names. + We want to see at least one such key, and for all such keys to be in 'already_built_manifest_jobs'. """ steps = circleci_job.get("steps", []) matching_steps = 0 for step in steps: - run_info = step.get("run", {}) + run_info = step.get("run", "") # Check if run_info is a string, short-hand notation if isinstance(run_info, str): - command = run_info - else: - command = run_info.get("command", "") - match = re.search(KNOWN_PREFIXES_PATTERN, command, re.IGNORECASE) - if not match: + # if there's no run key, or we use string short-hand, continue + continue + keys = run_info.get("aztec_manifest_key", []) + if isinstance(keys, str): + keys = [keys] + if not keys: # empty list? continue continue - matched_manifest = match.group(2) - if matched_manifest in already_built_manifest_jobs: - matching_steps += 1 - else: - # We have found a different string here - bail out - return False + for key in keys: + if key not in already_built_manifest_jobs: + # We have found a different string here - bail out + return False + matching_steps += 1 # All steps have matched - but make sure that's actually more than one step return matching_steps > 0 From 5e0b0d23ad913dbf20b1aab7f09e64a6755d6e45 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 14:58:27 +0000 Subject: [PATCH 26/47] add aztec keys to everything --- .circleci/config.yml | 82 ++++++++++++++++++- .../scripts/generate_circleci_config.py | 4 +- 2 files changed, 82 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 053fee95ab7..e39e4d60a04 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -151,6 +151,7 @@ jobs: - run: name: "Build" command: cond_spot_run_build noir-compile-acir-tests 32 + aztec_manifest_key: noir # for circleci build filtering avm-transpiler: docker: @@ -162,6 +163,7 @@ jobs: - run: name: "Build" command: cond_spot_run_build avm-transpiler 32 + aztec_manifest_key: avm-transpiler # for circleci build filtering # Barretenberg barretenberg-wasm-linux-clang: @@ -174,6 +176,7 @@ jobs: - run: name: "Build" command: cond_spot_run_build barretenberg-wasm-linux-clang 128 + aztec_manifest_key: barretenberg-wasm-linux-clang # for circleci build filtering barretenberg-x86_64-linux-gcc: docker: @@ -185,6 +188,7 @@ jobs: - run: name: "Build" command: cond_spot_run_build barretenberg-x86_64-linux-gcc 128 + aztec_manifest_key: barretenberg-x86_64-linux-gcc # for circleci build filtering barretenberg-x86_64-linux-clang: docker: @@ -196,6 +200,7 @@ jobs: - run: name: "Build" command: cond_spot_run_build barretenberg-x86_64-linux-clang 128 + aztec_manifest_key: barretenberg-x86_64-linux-clang barretenberg-x86_64-linux-clang-fuzzing: docker: @@ -207,7 +212,8 @@ jobs: - run: name: "Build" command: cond_spot_run_build barretenberg-x86_64-linux-clang-fuzzing 128 - + aztec_manifest_key: barretenberg-x86_64-linux-clang-fuzzing + barretenberg-x86_64-linux-clang-assert: docker: - image: aztecprotocol/alpine-build-image @@ -218,6 +224,7 @@ jobs: - run: name: "Build" command: cond_spot_run_build barretenberg-x86_64-linux-clang-assert 128 + aztec_manifest_key: barretenberg-x86_64-linux-clang-assert barretenberg-x86_64-linux-clang-sol: docker: @@ -229,6 +236,7 @@ jobs: - run: name: "Build" command: cond_spot_run_build barretenberg-x86_64-linux-clang-sol 32 + aztec_manifest_key: barretenberg-x86_64-linux-clang-sol barretenberg-docs: machine: @@ -240,6 +248,7 @@ jobs: - run: name: "Build barretenberg docs" command: build barretenberg-docs + aztec_manifest_key: barretenberg-docs - run: name: "Deploy barretenberg docs" command: | @@ -255,6 +264,7 @@ jobs: - run: name: "Test" command: cond_spot_run_test barretenberg-x86_64-linux-clang-assert 32 ./scripts/run_tests 1 stdlib-tests + aztec_manifest_key: barretenberg-x86_64-linux-clang-assert barretenberg-dsl-tests: docker: @@ -266,6 +276,7 @@ jobs: - run: name: "Test" command: cond_spot_run_test barretenberg-x86_64-linux-clang-assert 32 ./scripts/run_tests 1 dsl_tests + aztec_manifest_key: barretenberg-x86_64-linux-clang-assert barretenberg-tests: docker: @@ -277,6 +288,7 @@ jobs: - run: name: "Test" command: cond_spot_run_test barretenberg-x86_64-linux-clang-assert 32 ./scripts/bb-tests.sh + aztec_manifest_key: barretenberg-x86_64-linux-clang-assert barretenberg-bench: machine: @@ -290,6 +302,7 @@ jobs: - run: name: "Benchmark" command: cond_spot_run_build barretenberg-bench 32 + aztec_manifest_key: barretenberg-bench - run: name: "Upload" command: | @@ -305,6 +318,7 @@ jobs: - run: name: "Test" command: cond_spot_run_test barretenberg-x86_64-linux-clang-assert 32 ./scripts/run_tests 1 proof_system_tests + aztec_manifest_key: barretenberg-x86_64-linux-clang-assert barretenberg-stdlib-recursion-ultra-tests: docker: @@ -316,6 +330,7 @@ jobs: - run: name: "Test" command: cond_spot_run_test barretenberg-x86_64-linux-clang-assert 32 ./scripts/run_tests 3 stdlib_recursion_tests --gtest_filter=-*turbo* + aztec_manifest_key: barretenberg-x86_64-linux-clang-assert barretenberg-join-split-tests: docker: @@ -327,6 +342,7 @@ jobs: - run: name: "Test" command: cond_spot_run_test barretenberg-x86_64-linux-clang-assert 32 ./scripts/run_tests 3 join_split_example_proofs_join_split_tests --gtest_filter=-*full_proof* + aztec_manifest_key: barretenberg-x86_64-linux-clang-assert barretenberg-acir-tests-bb: docker: @@ -338,6 +354,7 @@ jobs: - run: name: "Build and test" command: cond_spot_run_build barretenberg-acir-tests-bb 32 + aztec_manifest_key: barretenberg-acir-tests-bb bb-js: machine: @@ -349,6 +366,7 @@ jobs: - run: name: "Build and test" command: build bb.js + aztec_manifest_key: bb.js bb-js-tests: docker: @@ -360,6 +378,7 @@ jobs: - run: name: "Build and test" command: cond_spot_run_test bb.js 32 ./scripts/run_tests + aztec_manifest_key: bb.js bb-js-acir-tests: docker: @@ -371,6 +390,7 @@ jobs: - run: name: "Build and test" command: cond_spot_run_build barretenberg-acir-tests-bb.js 32 + aztec_manifest_key: barretenberg-acir-tests-bb.js l1-contracts: machine: @@ -382,6 +402,7 @@ jobs: - run: name: "Build and test" command: build l1-contracts + aztec_manifest_key: l1-contracts noir-projects: machine: @@ -393,6 +414,7 @@ jobs: - run: name: "Build and test" command: build noir-projects + aztec_manifest_key: noir-projects boxes-files: machine: @@ -404,6 +426,7 @@ jobs: - run: name: "Build" command: build boxes-files + aztec_manifest_key: boxes-files yarn-project-base: machine: @@ -415,6 +438,7 @@ jobs: - run: name: "Build" command: build yarn-project-base | add_timestamps + aztec_manifest_key: yarn-project-base yarn-project: machine: @@ -426,6 +450,7 @@ jobs: - run: name: Build command: build yarn-project | add_timestamps + aztec_manifest_key: yarn-project yarn-project-prod: machine: @@ -437,6 +462,7 @@ jobs: - run: name: Build command: build yarn-project-prod | add_timestamps + aztec_manifest_key: yarn-project-prod yarn-project-formatting: docker: @@ -448,6 +474,7 @@ jobs: - run: name: Check Formatting command: cond_spot_run_container yarn-project 8 formatting | add_timestamps + aztec_manifest_key: yarn-project yarn-project-tests: docker: @@ -459,6 +486,7 @@ jobs: - run: name: Test command: cond_spot_run_container yarn-project 64 test | add_timestamps + aztec_manifest_key: yarn-project aztec-package: machine: @@ -470,6 +498,7 @@ jobs: - run: name: "Build and test" command: build aztec + aztec_manifest_key: aztec cli: machine: @@ -481,6 +510,7 @@ jobs: - run: name: "Build and test" command: build cli + aztec_manifest_key: cli mainnet-fork: machine: @@ -492,6 +522,7 @@ jobs: - run: name: "Build" command: build mainnet-fork | add_timestamps + aztec_manifest_key: mainnet-fork aztec-faucet: machine: @@ -503,6 +534,7 @@ jobs: - run: name: "Build and test" command: build aztec-faucet | add_timestamps + aztec_manifest_key: aztec-faucet boxes: docker: @@ -514,6 +546,7 @@ jobs: - run: name: "Build" command: cond_spot_run_build boxes 4 + aztec_manifest_key: boxes boxes-blank: docker: @@ -525,6 +558,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose boxes 4 ./docker-compose.yml BOX=box-blank + aztec_manifest_key: boxes boxes-blank-react: docker: @@ -536,6 +570,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose boxes 4 ./docker-compose.yml BOX=box-blank-react + aztec_manifest_key: boxes end-to-end: machine: @@ -547,6 +582,7 @@ jobs: - run: name: "Build" command: build end-to-end + aztec_manifest_key: end-to-end e2e-2-pxes: docker: @@ -558,6 +594,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_2_pxes.test.ts + aztec_manifest_key: end-to-end e2e-note-getter: docker: @@ -569,6 +606,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_note_getter.test.ts + aztec_manifest_key: end-to-end e2e-counter: docker: @@ -580,6 +618,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_counter_contract.test.ts + aztec_manifest_key: end-to-end e2e-private-voting: docker: @@ -591,6 +630,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_private_voting_contract.test.ts + aztec_manifest_key: end-to-end e2e-multiple-accounts-1-enc-key: docker: @@ -602,6 +642,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_multiple_accounts_1_enc_key.test.ts + aztec_manifest_key: end-to-end e2e-deploy-contract: docker: @@ -613,6 +654,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_deploy_contract.test.ts + aztec_manifest_key: end-to-end e2e-lending-contract: docker: @@ -624,6 +666,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_lending_contract.test.ts + aztec_manifest_key: end-to-end e2e-token-contract: docker: @@ -635,6 +678,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_token_contract.test.ts + aztec_manifest_key: end-to-end e2e-blacklist-token-contract: docker: @@ -646,6 +690,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_blacklist_token_contract.test.ts + aztec_manifest_key: end-to-end # TODO(3458): Investigate intermittent failure # e2e-slow-tree: @@ -658,6 +703,7 @@ jobs: # - run: # name: "Test" # command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_slow_tree.test.ts + # aztec_manifest_key: end-to-end e2e-sandbox-example: docker: @@ -669,6 +715,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_sandbox_example.test.ts + aztec_manifest_key: end-to-end e2e-state-vars: docker: @@ -680,6 +727,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_state_vars.test.ts + aztec_manifest_key: end-to-end e2e-block-building: docker: @@ -691,6 +739,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_block_building.test.ts + aztec_manifest_key: end-to-end e2e-nested-contract: docker: @@ -702,6 +751,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_nested_contract.test.ts + aztec_manifest_key: end-to-end e2e-static-calls: docker: @@ -713,6 +763,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_static_calls.test.ts + aztec_manifest_key: end-to-end e2e-delegate-calls: docker: @@ -724,6 +775,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_delegate_calls.test.ts + aztec_manifest_key: end-to-end e2e-non-contract-account: docker: @@ -735,6 +787,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_non_contract_account.test.ts + aztec_manifest_key: end-to-end e2e-cross-chain-messaging: docker: @@ -746,6 +799,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_cross_chain_messaging.test.ts + aztec_manifest_key: end-to-end e2e-public-cross-chain-messaging: docker: @@ -757,6 +811,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_public_cross_chain_messaging.test.ts + aztec_manifest_key: end-to-end e2e-public-to-private-messaging: docker: @@ -768,6 +823,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_public_to_private_messaging.test.ts + aztec_manifest_key: end-to-end e2e-account-contracts: docker: @@ -779,6 +835,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_account_contracts.test.ts + aztec_manifest_key: end-to-end e2e-escrow-contract: docker: @@ -790,6 +847,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_escrow_contract.test.ts + aztec_manifest_key: end-to-end e2e-inclusion-proofs-contract: docker: @@ -812,6 +870,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_pending_commitments_contract.test.ts + aztec_manifest_key: end-to-end e2e-ordering: docker: @@ -823,6 +882,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_ordering.test.ts + aztec_manifest_key: end-to-end uniswap-trade-on-l1-from-l2: docker: @@ -834,6 +894,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=uniswap_trade_on_l1_from_l2.test.ts + aztec_manifest_key: end-to-end integration-archiver-l1-to-l2: docker: @@ -845,6 +906,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=integration_archiver_l1_to_l2.test.ts + aztec_manifest_key: end-to-end integration-l1-publisher: docker: @@ -856,6 +918,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=integration_l1_publisher.test.ts + aztec_manifest_key: end-to-end e2e-cli: docker: @@ -867,6 +930,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_cli.test.ts + aztec_manifest_key: end-to-end e2e-persistence: docker: @@ -878,6 +942,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=e2e_persistence.test.ts + aztec_manifest_key: end-to-end e2e-browser: docker: @@ -889,6 +954,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-browser.yml TEST=e2e_aztec_js_browser.test.ts + aztec_manifest_key: end-to-end e2e-card-game: docker: @@ -900,6 +966,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_card_game.test.ts + aztec_manifest_key: end-to-end e2e-avm-simulator: docker: @@ -911,6 +978,7 @@ jobs: - run: name: "Test" command: AVM_ENABLED=1 cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_avm_simulator.test.ts + aztec_manifest_key: end-to-end e2e-fees: docker: @@ -922,6 +990,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_fees.test.ts + aztec_manifest_key: end-to-end pxe: docker: @@ -933,6 +1002,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=pxe_sandbox.test.ts + aztec_manifest_key: end-to-end cli-docs-sandbox: docker: @@ -944,6 +1014,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=cli_docs_sandbox.test.ts + aztec_manifest_key: end-to-end e2e-docs-examples: docker: @@ -955,6 +1026,7 @@ jobs: - run: name: "Test" command: AVM_ENABLED=1 cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=docs_examples_test.ts + aztec_manifest_key: end-to-end guides-writing-an-account-contract: docker: @@ -966,6 +1038,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=guides/writing_an_account_contract.test.ts + aztec_manifest_key: end-to-end guides-dapp-testing: docker: @@ -977,6 +1050,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=guides/dapp_testing.test.ts + aztec_manifest_key: end-to-end guides-sample-dapp: docker: @@ -988,6 +1062,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=sample-dapp + aztec_manifest_key: end-to-end guides-up-quick-start: docker: @@ -999,6 +1074,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=guides/up_quick_start.test.ts + aztec_manifest_key: end-to-end bench-publish-rollup: docker: @@ -1010,6 +1086,7 @@ jobs: - run: name: "Benchmark" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=benchmarks/bench_publish_rollup.test.ts DEBUG=aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees + aztec_manifest_key: end-to-end bench-process-history: docker: @@ -1021,6 +1098,7 @@ jobs: - run: name: "Benchmark" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=benchmarks/bench_process_history.test.ts DEBUG=aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees + aztec_manifest_key: end-to-end build-docs: machine: @@ -1046,6 +1124,7 @@ jobs: - run: name: "Build docs" command: build docs + aztec_manifest_key: docs - run: name: "Deploy docs" command: | @@ -1066,6 +1145,7 @@ jobs: - run: name: "Build yellow paper" command: build yellow-paper + aztec_manifest_key: yellow-paper e2e-join: docker: diff --git a/build-system/scripts/generate_circleci_config.py b/build-system/scripts/generate_circleci_config.py index 7bac7495566..67618dfb053 100755 --- a/build-system/scripts/generate_circleci_config.py +++ b/build-system/scripts/generate_circleci_config.py @@ -112,6 +112,4 @@ def eprint(*args, **kwargs): workflow_dict["workflows"]["system"]["when"] = {"equal":["system","<< pipeline.parameters.workflow >>"]} # Convert the new workflow back to JSON string new_workflow_json_str = json.dumps(workflow_dict, indent=2) - for t in workflow_dict["workflows"]["system"]["jobs"]: - eprint("KEPT", t) - # print(new_workflow_json_str) \ No newline at end of file + print(new_workflow_json_str) \ No newline at end of file From bc262614e51d44fc818ab3a79e59c57a0c59e752 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 15:05:10 +0000 Subject: [PATCH 27/47] redo with fixed ecr_login --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e39e4d60a04..c3d2ac2cffd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -82,7 +82,6 @@ jobs: - run: name: Generate Pipeline generated_config.yml file command: | - echo "$AWS_SECRET_ACCESS_KEY" # ability to query ECR images ecr_login # convert to json, run a python script, write out json (which is a readable as YAML) From 68eed43050cc54db502111434dbc2d04883e3416 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 15:08:01 +0000 Subject: [PATCH 28/47] fixes --- .circleci/config.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c3d2ac2cffd..f174ad2b8ec 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -73,9 +73,9 @@ setup_env: &setup_env jobs: # Dynamically filter our code, quickly figuring out which jobs we can skip. generate-config: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -84,6 +84,7 @@ jobs: command: | # ability to query ECR images ecr_login + set +eu # convert to json, run a python script, write out json (which is a readable as YAML) pip install PyYAML build-system/scripts/generate_circleci_config.py > .circleci/generated_config.yml From ad7f6904b603053a67ad737f42f65eb043aaa9e1 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 15:09:59 +0000 Subject: [PATCH 29/47] [ci debug] From 490a6afee401fb6911db15db608f8a1334213b95 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 15:11:09 +0000 Subject: [PATCH 30/47] [ci debug] test --- .circleci/config.yml | 3 ++- build-system/scripts/generate_circleci_config.py | 8 ++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f174ad2b8ec..9ff13529823 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -87,7 +87,8 @@ jobs: set +eu # convert to json, run a python script, write out json (which is a readable as YAML) pip install PyYAML - build-system/scripts/generate_circleci_config.py > .circleci/generated_config.yml + build-system/scripts/generate_circleci_config.py + #> .circleci/generated_config.yml cat .circleci/generated_config.yml echo "[]" > .circleci/generated_config.yml - continuation/continue: diff --git a/build-system/scripts/generate_circleci_config.py b/build-system/scripts/generate_circleci_config.py index 67618dfb053..ba853a3f7c2 100755 --- a/build-system/scripts/generate_circleci_config.py +++ b/build-system/scripts/generate_circleci_config.py @@ -88,10 +88,6 @@ def remove_jobs_from_workflow(jobs, to_remove): job[key]["requires"] = [r for r in job[key].get("requires", []) if r not in jobs_to_remove] new_jobs.append(job) return new_jobs -import sys - -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, **kwargs) if __name__ == '__main__': # The CircleCI workflow as a JSON string (Replace this with your actual workflow) @@ -102,7 +98,7 @@ def eprint(*args, **kwargs): # # List of jobs to remove jobs_to_remove = list(get_already_built_circleci_job_names(workflow_dict["jobs"])) for key in jobs_to_remove: - eprint("KEY", key) + print("KEY", key) # Get rid of workflow setup step and setup flag workflow_dict["setup"] = False @@ -112,4 +108,4 @@ def eprint(*args, **kwargs): workflow_dict["workflows"]["system"]["when"] = {"equal":["system","<< pipeline.parameters.workflow >>"]} # Convert the new workflow back to JSON string new_workflow_json_str = json.dumps(workflow_dict, indent=2) - print(new_workflow_json_str) \ No newline at end of file + # print(new_workflow_json_str) \ No newline at end of file From 9659ab7eeb82944503badf36dcd2d2e209b4b78d Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 15:13:35 +0000 Subject: [PATCH 31/47] test --- build-system/scripts/generate_circleci_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build-system/scripts/generate_circleci_config.py b/build-system/scripts/generate_circleci_config.py index ba853a3f7c2..e325fdb0ba8 100755 --- a/build-system/scripts/generate_circleci_config.py +++ b/build-system/scripts/generate_circleci_config.py @@ -90,6 +90,7 @@ def remove_jobs_from_workflow(jobs, to_remove): return new_jobs if __name__ == '__main__': + print("HI!") # The CircleCI workflow as a JSON string (Replace this with your actual workflow) # Convert the JSON string to a Python dictionary From b4ad27656690f6aa1c2298dd1e41e65c6530a02d Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 15:14:44 +0000 Subject: [PATCH 32/47] test [ci debug] --- .circleci/config.yml | 1 + build-system/scripts/ecr_login | 1 + 2 files changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9ff13529823..b0cf4766a6a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -85,6 +85,7 @@ jobs: # ability to query ECR images ecr_login set +eu + set -x # convert to json, run a python script, write out json (which is a readable as YAML) pip install PyYAML build-system/scripts/generate_circleci_config.py diff --git a/build-system/scripts/ecr_login b/build-system/scripts/ecr_login index 5d9e6671e59..65180dc1ad6 100755 --- a/build-system/scripts/ecr_login +++ b/build-system/scripts/ecr_login @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euo pipefail REGION=${1:-$ECR_REGION} +aws ecr get-login-password --region $REGION aws ecr get-login-password --region $REGION \ | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$REGION.amazonaws.com 2> /dev/null \ No newline at end of file From dacea603d4f01babe07e15a8d34549eb02605bea Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 15:27:38 +0000 Subject: [PATCH 33/47] better print --- build-system/scripts/ecr_login | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-system/scripts/ecr_login b/build-system/scripts/ecr_login index 65180dc1ad6..e26c2d3153c 100755 --- a/build-system/scripts/ecr_login +++ b/build-system/scripts/ecr_login @@ -1,6 +1,6 @@ #!/usr/bin/env bash set -euo pipefail REGION=${1:-$ECR_REGION} -aws ecr get-login-password --region $REGION aws ecr get-login-password --region $REGION \ - | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$REGION.amazonaws.com 2> /dev/null \ No newline at end of file + | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$REGION.amazonaws.com + #2> /dev/null \ No newline at end of file From cfb06f5f35fdfed90061f7f578a466226dbdff57 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 15:35:33 +0000 Subject: [PATCH 34/47] revert --- .circleci/config.yml | 4 ---- build-system/scripts/ecr_login | 3 +-- build-system/scripts/generate_circleci_config.py | 1 - 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b0cf4766a6a..df5dc12d1ee 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -82,10 +82,6 @@ jobs: - run: name: Generate Pipeline generated_config.yml file command: | - # ability to query ECR images - ecr_login - set +eu - set -x # convert to json, run a python script, write out json (which is a readable as YAML) pip install PyYAML build-system/scripts/generate_circleci_config.py diff --git a/build-system/scripts/ecr_login b/build-system/scripts/ecr_login index e26c2d3153c..5d9e6671e59 100755 --- a/build-system/scripts/ecr_login +++ b/build-system/scripts/ecr_login @@ -2,5 +2,4 @@ set -euo pipefail REGION=${1:-$ECR_REGION} aws ecr get-login-password --region $REGION \ - | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$REGION.amazonaws.com - #2> /dev/null \ No newline at end of file + | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$REGION.amazonaws.com 2> /dev/null \ No newline at end of file diff --git a/build-system/scripts/generate_circleci_config.py b/build-system/scripts/generate_circleci_config.py index e325fdb0ba8..ba853a3f7c2 100755 --- a/build-system/scripts/generate_circleci_config.py +++ b/build-system/scripts/generate_circleci_config.py @@ -90,7 +90,6 @@ def remove_jobs_from_workflow(jobs, to_remove): return new_jobs if __name__ == '__main__': - print("HI!") # The CircleCI workflow as a JSON string (Replace this with your actual workflow) # Convert the JSON string to a Python dictionary From e34259b84370e21b4eda737f7bc03dd72c475774 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 15:37:10 +0000 Subject: [PATCH 35/47] better print --- build-system/scripts/generate_circleci_config.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/build-system/scripts/generate_circleci_config.py b/build-system/scripts/generate_circleci_config.py index ba853a3f7c2..79e5348e617 100755 --- a/build-system/scripts/generate_circleci_config.py +++ b/build-system/scripts/generate_circleci_config.py @@ -88,6 +88,10 @@ def remove_jobs_from_workflow(jobs, to_remove): job[key]["requires"] = [r for r in job[key].get("requires", []) if r not in jobs_to_remove] new_jobs.append(job) return new_jobs +import sys + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) if __name__ == '__main__': # The CircleCI workflow as a JSON string (Replace this with your actual workflow) @@ -98,7 +102,7 @@ def remove_jobs_from_workflow(jobs, to_remove): # # List of jobs to remove jobs_to_remove = list(get_already_built_circleci_job_names(workflow_dict["jobs"])) for key in jobs_to_remove: - print("KEY", key) + eprint("KEY", key) # Get rid of workflow setup step and setup flag workflow_dict["setup"] = False From 1cc46e1b1fd9cbed0cf25adfa3cd82ce45dffa24 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 15:38:00 +0000 Subject: [PATCH 36/47] better print --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index df5dc12d1ee..0b3858cb50d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -75,7 +75,7 @@ jobs: generate-config: docker: - image: aztecprotocol/alpine-build-image - resource_class: small + resource_class: xlarge steps: - *checkout - *setup_env From 3b42f1095b9b069a776d696158e29c30a99d1795 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 15:38:58 +0000 Subject: [PATCH 37/47] productionize --- .circleci/config.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0b3858cb50d..5c9efb47d47 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -84,10 +84,7 @@ jobs: command: | # convert to json, run a python script, write out json (which is a readable as YAML) pip install PyYAML - build-system/scripts/generate_circleci_config.py - #> .circleci/generated_config.yml - cat .circleci/generated_config.yml - echo "[]" > .circleci/generated_config.yml + build-system/scripts/generate_circleci_config.py > .circleci/generated_config.yml - continuation/continue: configuration_path: .circleci/generated_config.yml # Noir From aca0cb2487df91fd6d0a3892128853c2f7820d1e Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 15:41:14 +0000 Subject: [PATCH 38/47] reinstate config --- build-system/scripts/generate_circleci_config.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/build-system/scripts/generate_circleci_config.py b/build-system/scripts/generate_circleci_config.py index 79e5348e617..9a6bc27c0b4 100755 --- a/build-system/scripts/generate_circleci_config.py +++ b/build-system/scripts/generate_circleci_config.py @@ -8,6 +8,10 @@ import re from concurrent.futures import ProcessPoolExecutor, as_completed import subprocess +import sys + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) # same functionality as query_manifest rebuildPatterns but in bulk def get_manifest_job_names(): @@ -88,10 +92,6 @@ def remove_jobs_from_workflow(jobs, to_remove): job[key]["requires"] = [r for r in job[key].get("requires", []) if r not in jobs_to_remove] new_jobs.append(job) return new_jobs -import sys - -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, **kwargs) if __name__ == '__main__': # The CircleCI workflow as a JSON string (Replace this with your actual workflow) @@ -102,7 +102,7 @@ def eprint(*args, **kwargs): # # List of jobs to remove jobs_to_remove = list(get_already_built_circleci_job_names(workflow_dict["jobs"])) for key in jobs_to_remove: - eprint("KEY", key) + eprint("SKIPPING BUILT MANIFEST KEY", key) # Get rid of workflow setup step and setup flag workflow_dict["setup"] = False @@ -112,4 +112,4 @@ def eprint(*args, **kwargs): workflow_dict["workflows"]["system"]["when"] = {"equal":["system","<< pipeline.parameters.workflow >>"]} # Convert the new workflow back to JSON string new_workflow_json_str = json.dumps(workflow_dict, indent=2) - # print(new_workflow_json_str) \ No newline at end of file + print(new_workflow_json_str) \ No newline at end of file From 0556f3861a73f5da23d67aa9708d346d7a19acb1 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 15:43:49 +0000 Subject: [PATCH 39/47] cleanup --- .circleci/config.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5c9efb47d47..b508d9b3717 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -98,7 +98,7 @@ jobs: - run: name: "Build" command: cond_spot_run_build noir 32 - aztec_manifest_key: noir # for circleci build filtering + aztec_manifest_key: noir noir-arm64: docker: @@ -110,7 +110,7 @@ jobs: - run: name: "Build" command: cond_spot_run_build noir 32 arm64 - aztec_manifest_key: noir # for circleci build filtering + aztec_manifest_key: noir noir-ecr-manifest: machine: @@ -122,7 +122,7 @@ jobs: - run: name: "Create ECR manifest" command: create_ecr_manifest noir x86_64,arm64 - aztec_manifest_key: noir # for circleci build filtering + aztec_manifest_key: noir noir-packages: docker: @@ -134,7 +134,7 @@ jobs: - run: name: "Build" command: cond_spot_run_build noir-packages 32 - aztec_manifest_key: noir-packages # for circleci build filtering + aztec_manifest_key: noir-packages noir-compile-acir-tests: docker: @@ -146,7 +146,7 @@ jobs: - run: name: "Build" command: cond_spot_run_build noir-compile-acir-tests 32 - aztec_manifest_key: noir # for circleci build filtering + aztec_manifest_key: noir avm-transpiler: docker: @@ -158,7 +158,7 @@ jobs: - run: name: "Build" command: cond_spot_run_build avm-transpiler 32 - aztec_manifest_key: avm-transpiler # for circleci build filtering + aztec_manifest_key: avm-transpiler # Barretenberg barretenberg-wasm-linux-clang: @@ -171,7 +171,7 @@ jobs: - run: name: "Build" command: cond_spot_run_build barretenberg-wasm-linux-clang 128 - aztec_manifest_key: barretenberg-wasm-linux-clang # for circleci build filtering + aztec_manifest_key: barretenberg-wasm-linux-clang barretenberg-x86_64-linux-gcc: docker: @@ -183,7 +183,7 @@ jobs: - run: name: "Build" command: cond_spot_run_build barretenberg-x86_64-linux-gcc 128 - aztec_manifest_key: barretenberg-x86_64-linux-gcc # for circleci build filtering + aztec_manifest_key: barretenberg-x86_64-linux-gcc barretenberg-x86_64-linux-clang: docker: @@ -854,6 +854,7 @@ jobs: - run: name: "Test" command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_inclusion_proofs_contract.test.ts + aztec_manifest_key: end-to-end e2e-pending-commitments-contract: docker: From 1c2821c722978c1686a759ff9340f14799e80bee Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 15:58:18 +0000 Subject: [PATCH 40/47] fix rebuild patterns --- build_manifest.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/build_manifest.yml b/build_manifest.yml index bac4b7bc86e..665e16f6b24 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -207,11 +207,11 @@ docs: dockerfile: docs/Dockerfile rebuildPatterns: - ^docs/ - - ^.*.cpp$ - - ^.*.hpp$ - - ^.*.ts$ - - ^.release-please-manifest.json$ - - ^.*.nr$ + - ^.*\.cpp$ + - ^.*\.hpp$ + - ^.*\.ts$ + - ^.release-please-manifest\.json$ + - ^.*\.nr$ dependencies: - yarn-project - noir-projects From 1a6c1fe09be8781aae5cbeb5947725d171e5ae6a Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 16:07:39 +0000 Subject: [PATCH 41/47] more printing --- .circleci/config.yml | 2 +- build-system/scripts/generate_circleci_config.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b508d9b3717..92fc9dfd303 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -82,7 +82,7 @@ jobs: - run: name: Generate Pipeline generated_config.yml file command: | - # convert to json, run a python script, write out json (which is a readable as YAML) + # filter our circleci config to the minimal pipeline pip install PyYAML build-system/scripts/generate_circleci_config.py > .circleci/generated_config.yml - continuation/continue: diff --git a/build-system/scripts/generate_circleci_config.py b/build-system/scripts/generate_circleci_config.py index 9a6bc27c0b4..38e57ed05b1 100755 --- a/build-system/scripts/generate_circleci_config.py +++ b/build-system/scripts/generate_circleci_config.py @@ -47,6 +47,8 @@ def is_already_built_circleci_job(circleci_job, already_built_manifest_jobs): def get_already_built_circleci_job_names(circleci_jobs): already_built_manifest_jobs = list(get_already_built_manifest_job_names()) + for key in jobs_to_remove: + eprint("Detected cached manifest key:", key) for job_name, circleci_job in circleci_jobs.items(): if is_already_built_circleci_job(circleci_job, already_built_manifest_jobs): yield job_name @@ -102,7 +104,7 @@ def remove_jobs_from_workflow(jobs, to_remove): # # List of jobs to remove jobs_to_remove = list(get_already_built_circleci_job_names(workflow_dict["jobs"])) for key in jobs_to_remove: - eprint("SKIPPING BUILT MANIFEST KEY", key) + eprint("Skipping circleci job", key) # Get rid of workflow setup step and setup flag workflow_dict["setup"] = False From 9a796aaa21e0bed4334c797b690238a0a66ac9ef Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 16:09:07 +0000 Subject: [PATCH 42/47] fix --- build-system/scripts/generate_circleci_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-system/scripts/generate_circleci_config.py b/build-system/scripts/generate_circleci_config.py index 38e57ed05b1..5d3d5c50169 100755 --- a/build-system/scripts/generate_circleci_config.py +++ b/build-system/scripts/generate_circleci_config.py @@ -47,7 +47,7 @@ def is_already_built_circleci_job(circleci_job, already_built_manifest_jobs): def get_already_built_circleci_job_names(circleci_jobs): already_built_manifest_jobs = list(get_already_built_manifest_job_names()) - for key in jobs_to_remove: + for key in already_built_manifest_jobs: eprint("Detected cached manifest key:", key) for job_name, circleci_job in circleci_jobs.items(): if is_already_built_circleci_job(circleci_job, already_built_manifest_jobs): From 12a4fd7e28419563bf7b67aef045fea2792414e1 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 16:10:29 +0000 Subject: [PATCH 43/47] try not using xlarge --- .circleci/config.yml | 2 +- build-system/scripts/generate_circleci_config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 92fc9dfd303..3baa3588bef 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -75,7 +75,7 @@ jobs: generate-config: docker: - image: aztecprotocol/alpine-build-image - resource_class: xlarge + resource_class: large steps: - *checkout - *setup_env diff --git a/build-system/scripts/generate_circleci_config.py b/build-system/scripts/generate_circleci_config.py index 5d3d5c50169..f04328646d0 100755 --- a/build-system/scripts/generate_circleci_config.py +++ b/build-system/scripts/generate_circleci_config.py @@ -104,7 +104,7 @@ def remove_jobs_from_workflow(jobs, to_remove): # # List of jobs to remove jobs_to_remove = list(get_already_built_circleci_job_names(workflow_dict["jobs"])) for key in jobs_to_remove: - eprint("Skipping circleci job", key) + eprint("Skipping circleci job:", key) # Get rid of workflow setup step and setup flag workflow_dict["setup"] = False From 2d927919b0fac43edb40c5040fd0dde725fea98a Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 16:14:42 +0000 Subject: [PATCH 44/47] seems worth saving 8 seconds? --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1c863768f99..fcd68141cc4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -75,7 +75,7 @@ jobs: generate-config: docker: - image: aztecprotocol/alpine-build-image - resource_class: large + resource_class: xlarge steps: - *checkout - *setup_env From 3685d1abe09ef779ffb1d418cea6315553025187 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 16:17:00 +0000 Subject: [PATCH 45/47] revert. test --- build-system/scripts/image_exists | 1 - yarn-project/cli/src/index.ts | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/build-system/scripts/image_exists b/build-system/scripts/image_exists index 6e142d7cfb2..0ad9d90ae69 100755 --- a/build-system/scripts/image_exists +++ b/build-system/scripts/image_exists @@ -1,5 +1,4 @@ #!/usr/bin/env bash -[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu # Returns true if the given image exists in the current ECR. aws ecr describe-images --region=$ECR_REGION --repository-name=$1 --image-ids=imageTag=$2 > /dev/null 2>&1 diff --git a/yarn-project/cli/src/index.ts b/yarn-project/cli/src/index.ts index f5ccd6165c1..df792931937 100644 --- a/yarn-project/cli/src/index.ts +++ b/yarn-project/cli/src/index.ts @@ -24,6 +24,7 @@ import { parseTxHash, } from './parse_args.js'; +// TEST /** * If we can successfully resolve 'host.docker.internal', then we are running in a container, and we should treat * localhost as being host.docker.internal. From 00c3fea739e7e9c10ed8bfd6eba7992b3b9d3074 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 16:18:46 +0000 Subject: [PATCH 46/47] revert. test worked. --- yarn-project/cli/src/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/yarn-project/cli/src/index.ts b/yarn-project/cli/src/index.ts index df792931937..f5ccd6165c1 100644 --- a/yarn-project/cli/src/index.ts +++ b/yarn-project/cli/src/index.ts @@ -24,7 +24,6 @@ import { parseTxHash, } from './parse_args.js'; -// TEST /** * If we can successfully resolve 'host.docker.internal', then we are running in a container, and we should treat * localhost as being host.docker.internal. From d5203fea7334d0fb0edf901a1033aa597ce67a3b Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 23 Feb 2024 16:19:38 +0000 Subject: [PATCH 47/47] dont need to run pip --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fcd68141cc4..eb276b39bdf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -83,7 +83,6 @@ jobs: name: Generate Pipeline generated_config.yml file command: | # filter our circleci config to the minimal pipeline - pip install PyYAML build-system/scripts/generate_circleci_config.py > .circleci/generated_config.yml - continuation/continue: configuration_path: .circleci/generated_config.yml