diff --git a/.circleci/config.yml b/.circleci/config.yml index 693ef1710b7..3f908e5050b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -86,6 +86,7 @@ jobs: build-system/scripts/generate_circleci_config.py > .circleci/generated_config.yml - continuation/continue: configuration_path: .circleci/generated_config.yml + # Noir noir-x86_64: docker: @@ -207,7 +208,7 @@ jobs: name: "Build" command: cond_spot_run_build barretenberg-x86_64-linux-clang-fuzzing 128 aztec_manifest_key: barretenberg-x86_64-linux-clang-fuzzing - + barretenberg-x86_64-linux-clang-assert: docker: - image: aztecprotocol/alpine-build-image diff --git a/avm-transpiler/bootstrap.sh b/avm-transpiler/bootstrap.sh index 2c726cb4e43..e3446b3e6f1 100755 --- a/avm-transpiler/bootstrap.sh +++ b/avm-transpiler/bootstrap.sh @@ -17,6 +17,6 @@ if [ -n "$CMD" ]; then fi # Attempt to just pull artefacts from CI and exit on success. -./bootstrap_cache.sh && exit +[ -n "${USE_CACHE:-}" ] && ./bootstrap_cache.sh && exit ./scripts/bootstrap_native.sh \ No newline at end of file diff --git a/avm-transpiler/bootstrap_cache.sh b/avm-transpiler/bootstrap_cache.sh index dafe9ef5f0a..080136e2be8 100755 --- a/avm-transpiler/bootstrap_cache.sh +++ b/avm-transpiler/bootstrap_cache.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash set -eu -[ -z "${NO_CACHE:-}" ] && type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 - cd "$(dirname "$0")" source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null diff --git a/barretenberg/acir_tests/Dockerfile.bb b/barretenberg/acir_tests/Dockerfile.bb index 006db0e5333..4369c5f1033 100644 --- a/barretenberg/acir_tests/Dockerfile.bb +++ b/barretenberg/acir_tests/Dockerfile.bb @@ -1,5 +1,5 @@ -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/barretenberg-x86_64-linux-clang-assert -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-compile-acir-tests as noir-acir-tests +FROM aztecprotocol/barretenberg-x86_64-linux-clang-assert +FROM aztecprotocol/noir-compile-acir-tests as noir-acir-tests FROM node:18.19.0-alpine RUN apk update && apk add git bash curl jq coreutils @@ -10,7 +10,7 @@ COPY . . # Run every acir test through native bb build prove_then_verify flow for UltraPlonk. # This ensures we test independent pk construction through real/garbage witness data paths. RUN FLOW=prove_then_verify ./run_acir_tests.sh -# This flow is essentially the GoblinUltraHonk equivalent to the UltraPlonk "prove and verify". (This functionality is +# This flow is essentially the GoblinUltraHonk equivalent to the UltraPlonk "prove and verify". (This functionality is # accessed via the goblin "accumulate" mechanism). RUN FLOW=accumulate_and_verify_goblin ./run_acir_tests.sh # This is a "full" Goblin flow. It constructs and verifies four proofs: GoblinUltraHonk, ECCVM, Translator, and merge diff --git a/barretenberg/acir_tests/Dockerfile.bb.js b/barretenberg/acir_tests/Dockerfile.bb.js index b894826a5c2..ec6bac811da 100644 --- a/barretenberg/acir_tests/Dockerfile.bb.js +++ b/barretenberg/acir_tests/Dockerfile.bb.js @@ -1,5 +1,5 @@ -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/bb.js -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-compile-acir-tests as noir-acir-tests +FROM aztecprotocol/bb.js +FROM aztecprotocol/noir-compile-acir-tests as noir-acir-tests FROM node:18.19.0 COPY --from=0 /usr/src/barretenberg/ts-build /usr/src/barretenberg/ts diff --git a/barretenberg/acir_tests/Dockerfile.bb.sol b/barretenberg/acir_tests/Dockerfile.bb.sol index fe87fffc51e..dddb7e122f9 100644 --- a/barretenberg/acir_tests/Dockerfile.bb.sol +++ b/barretenberg/acir_tests/Dockerfile.bb.sol @@ -1,6 +1,6 @@ -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/barretenberg-x86_64-linux-clang-assert -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/barretenberg-x86_64-linux-clang-sol -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-compile-acir-tests as noir-acir-tests +FROM aztecprotocol/barretenberg-x86_64-linux-clang-assert +FROM aztecprotocol/barretenberg-x86_64-linux-clang-sol +FROM aztecprotocol/noir-compile-acir-tests as noir-acir-tests FROM node:18.19.0-alpine RUN apk update && apk add git bash curl jq diff --git a/barretenberg/acir_tests/Dockerfile.noir_acir_tests b/barretenberg/acir_tests/Dockerfile.noir_acir_tests index 5131e94e279..8fd78de30dc 100644 --- a/barretenberg/acir_tests/Dockerfile.noir_acir_tests +++ b/barretenberg/acir_tests/Dockerfile.noir_acir_tests @@ -2,7 +2,7 @@ # However, the noir subrepo has no concept of the aztec build pipeline, so the Dockerfile wouldn't make sense there. # So, it lives here. # This chains off the nargo build, and creates a container with a compiled set of acir tests. -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir +FROM aztecprotocol/noir RUN apt update && apt install -y jq && rm -rf /var/lib/apt/lists/* && apt-get clean ENV PATH="/usr/src/noir/target/release:${PATH}" WORKDIR /usr/src/noir/test_programs diff --git a/barretenberg/cpp/bootstrap.sh b/barretenberg/cpp/bootstrap.sh index 32de0856a4d..80ee4024b63 100755 --- a/barretenberg/cpp/bootstrap.sh +++ b/barretenberg/cpp/bootstrap.sh @@ -31,8 +31,11 @@ fi # Download ignition transcripts. (cd ./srs_db && ./download_ignition.sh 0) +# Install wasi-sdk. +./scripts/install-wasi-sdk.sh + # Attempt to just pull artefacts from CI and exit on success. -./bootstrap_cache.sh && exit +[ -n "${USE_CACHE:-}" ] && ./bootstrap_cache.sh && exit # Pick native toolchain file. ARCH=$(uname -m) @@ -54,23 +57,34 @@ echo "# Building with preset: $PRESET" echo "# When running cmake directly, remember to use: --build --preset $PRESET" echo "#################################" -# Build native. -cmake --preset $PRESET -DCMAKE_BUILD_TYPE=RelWithAssert -cmake --build --preset $PRESET --target bb +function build_native { + cmake --preset $PRESET -DCMAKE_BUILD_TYPE=RelWithAssert + cmake --build --preset $PRESET --target bb +} + +function build_wasm { + cmake --preset wasm + cmake --build --preset wasm +} + +function build_wasm_threads { + cmake --preset wasm-threads + cmake --build --preset wasm-threads +} + +g="\033[32m" # Green +b="\033[34m" # Blue +p="\033[35m" # Purple +r="\033[0m" # Reset + +(build_native > >(awk -v g="$g" -v r="$r" '$0=g"native: "r $0')) & +(build_wasm > >(awk -v b="$b" -v r="$r" '$0=b"wasm: "r $0')) & +(build_wasm_threads > >(awk -v p="$p" -v r="$r" '$0=p"wasm_threads: "r $0')) & + +wait if [ ! -d ./srs_db/grumpkin ]; then # The Grumpkin SRS is generated manually at the moment, only up to a large enough size for tests # If tests require more points, the parameter can be increased here. - (cd ./build && cmake --build . --parallel --target grumpkin_srs_gen && ./bin/grumpkin_srs_gen 8192) + cd ./build && cmake --build . --parallel --target grumpkin_srs_gen && ./bin/grumpkin_srs_gen 8192 fi - -# Install wasi-sdk. -./scripts/install-wasi-sdk.sh - -# Build WASM. -cmake --preset wasm -cmake --build --preset wasm - -# Build WASM with new threading. -cmake --preset wasm-threads -cmake --build --preset wasm-threads diff --git a/barretenberg/cpp/bootstrap_cache.sh b/barretenberg/cpp/bootstrap_cache.sh index 13e29707e3e..a739a6cab66 100755 --- a/barretenberg/cpp/bootstrap_cache.sh +++ b/barretenberg/cpp/bootstrap_cache.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash set -eu -[ -z "${NO_CACHE:-}" ] && type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 - cd "$(dirname "$0")" source ../../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null diff --git a/barretenberg/cpp/dockerfiles/Dockerfile.bench b/barretenberg/cpp/dockerfiles/Dockerfile.bench index 0535d2fb3a9..42226685814 100644 --- a/barretenberg/cpp/dockerfiles/Dockerfile.bench +++ b/barretenberg/cpp/dockerfiles/Dockerfile.bench @@ -1,4 +1,4 @@ -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/barretenberg-x86_64-linux-clang +FROM aztecprotocol/barretenberg-x86_64-linux-clang WORKDIR /usr/src/barretenberg/cpp RUN apk update && apk add curl libstdc++ jq RUN ./scripts/ci/ultra_honk_bench.sh diff --git a/barretenberg/ts/Dockerfile b/barretenberg/ts/Dockerfile index d28b5661b2d..855dc17be37 100644 --- a/barretenberg/ts/Dockerfile +++ b/barretenberg/ts/Dockerfile @@ -1,4 +1,4 @@ -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/barretenberg-wasm-linux-clang +FROM aztecprotocol/barretenberg-wasm-linux-clang FROM node:18.19.0-alpine COPY --from=0 /usr/src/barretenberg /usr/src/barretenberg diff --git a/barretenberg/ts/bootstrap.sh b/barretenberg/ts/bootstrap.sh index f8f4dfd56b8..e503cd4b52b 100755 --- a/barretenberg/ts/bootstrap.sh +++ b/barretenberg/ts/bootstrap.sh @@ -19,7 +19,7 @@ if [ -n "$CMD" ]; then fi # Attempt to just pull artefacts from CI and exit on success. -./bootstrap_cache.sh && exit +[ -n "${USE_CACHE:-}" ] && ./bootstrap_cache.sh && exit yarn install --immutable echo "Building with command 'yarn $BUILD_CMD'..." diff --git a/barretenberg/ts/bootstrap_cache.sh b/barretenberg/ts/bootstrap_cache.sh index c936d8cc7f5..c5e39052643 100755 --- a/barretenberg/ts/bootstrap_cache.sh +++ b/barretenberg/ts/bootstrap_cache.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash set -eu -[ -z "${NO_CACHE:-}" ] && type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 - cd "$(dirname "$0")" source ../../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null diff --git a/bootstrap.sh b/bootstrap.sh index adb97e67c76..e6d67f21f65 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -12,39 +12,47 @@ cd "$(dirname "$0")" CMD=${1:-} -if [ -n "$CMD" ]; then - if [ "$CMD" = "clean" ]; then - echo "WARNING: This will erase *all* untracked files, including hooks and submodules." - echo -n "Continue? [y/n] " - read user_input - if [ "$user_input" != "y" ] && [ "$user_input" != "Y" ]; then - exit 1 - fi +YELLOW="\033[93m" +BOLD="\033[1m" +RESET="\033[0m" - # Remove hooks and submodules. - rm -rf .git/hooks/* - rm -rf .git/modules/* - for SUBMODULE in $(git config --file .gitmodules --get-regexp path | awk '{print $2}'); do - rm -rf $SUBMODULE - done +source ./build-system/scripts/setup_env '' '' '' > /dev/null - # Remove all untracked files, directories, nested repos, and .gitignore files. - git clean -ffdx +if [ "$CMD" = "clean" ]; then + echo "WARNING: This will erase *all* untracked files, including hooks and submodules." + echo -n "Continue? [y/n] " + read user_input + if [ "$user_input" != "y" ] && [ "$user_input" != "Y" ]; then + exit 1 + fi + + # Remove hooks and submodules. + rm -rf .git/hooks/* + rm -rf .git/modules/* + for SUBMODULE in $(git config --file .gitmodules --get-regexp path | awk '{print $2}'); do + rm -rf $SUBMODULE + done - exit 0 - elif [ "$CMD" = "full" ]; then - export NO_CACHE=1 - else - echo "Unknown command: $CMD" + # Remove all untracked files, directories, nested repos, and .gitignore files. + git clean -ffdx + + exit 0 +elif [ "$CMD" = "full" ]; then + if can_use_ci_cache; then + echo -e "${BOLD}${YELLOW}WARNING: Performing a full bootstrap. Consider leveraging './bootstrap.sh fast' to use CI cache.${RESET}" + echo + fi +elif [ "$CMD" = "fast" ]; then + export USE_CACHE=1 + if ! can_use_ci_cache; then + echo -e "${BOLD}${YELLOW}WARNING: Either docker or aws credentials are missing. Install docker and request credentials. Note this is for internal aztec devs only.${RESET}" exit 1 fi +else + echo "usage: $0 " + exit 1 fi -# if [ ! -f ~/.nvm/nvm.sh ]; then -# echo "Nvm not found at ~/.nvm" -# exit 1 -# fi - # Install pre-commit git hooks. HOOKS_DIR=$(git rev-parse --git-path hooks) echo "(cd barretenberg/cpp && ./format.sh staged)" >$HOOKS_DIR/pre-commit diff --git a/boxes/Dockerfile b/boxes/Dockerfile index ef381f2fff5..921887d45ef 100644 --- a/boxes/Dockerfile +++ b/boxes/Dockerfile @@ -1,8 +1,8 @@ # Builds the boxes (they were copied into yarn-project-base so the cli can unbox). # Produces a container that can be run to test a specific box. See docker-compose.yml. -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/aztec AS aztec -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir as noir -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-projects as noir-projects +FROM aztecprotocol/aztec AS aztec +FROM aztecprotocol/noir as noir +FROM aztecprotocol/noir-projects as noir-projects # We need yarn. Start fresh container. FROM node:18.19.0 diff --git a/build-system/scripts/build b/build-system/scripts/build index e7b73b03e06..5ff4c289e39 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -104,14 +104,14 @@ for PARENT_REPO in $(query_manifest dependencies $REPOSITORY); do # Finally attempt to locate x86_64 image tag, as sometimes we build arch specific images from x86_64 images. PARENT_IMAGE_TAG=$(calculate_image_tag $PARENT_REPO x86_64) if ! image_exists $PARENT_REPO $PARENT_IMAGE_TAG; then - echo "Failed to locate multiarch image, arch specific image, or x86_64 image. Aborting." + echo "Failed to locate multiarch image, arch specific image, or x86_64 image for $PARENT_REPO. Aborting." exit 1 fi fi fi # Substitute references to parent repo, with the relevent built image uri. - DEPLOY_URI=$ECR_DEPLOY_URL/$PARENT_REPO + DEPLOY_URI=aztecprotocol/$PARENT_REPO PARENT_IMAGE_URI=$ECR_URL/$PARENT_REPO:$PARENT_IMAGE_TAG awk '{if ($1 == "FROM" && $2 == "'$DEPLOY_URI'") $2 = "'$PARENT_IMAGE_URI'"; print $0}' $DOCKERFILE > _temp && mv _temp $DOCKERFILE done diff --git a/build-system/scripts/build_local b/build-system/scripts/build_local index 8f4121dc5e7..2d54e6e0bce 100755 --- a/build-system/scripts/build_local +++ b/build-system/scripts/build_local @@ -64,8 +64,7 @@ for E in "${PROJECTS[@]}"; do fi fi - # TODO: Move everything to aztecprotocol/* - DEPLOY_IMAGE_URI=$ECR_DEPLOY_URL/$REPO:latest + DEPLOY_IMAGE_URI=aztecprotocol/$REPO:latest CACHE_IMAGE_URI=$(calculate_image_uri $REPO) ARR=(${CACHE_IMAGE_URI//:/ }) REPO_URI=${ARR[0]} @@ -112,9 +111,6 @@ for E in "${PROJECTS[@]}"; do # Retag with deployment uri. docker tag $CACHE_IMAGE_URI $DEPLOY_IMAGE_URI - # Retag for aztecprotocol dockerhub. - docker tag $DEPLOY_IMAGE_URI aztecprotocol/$REPO:latest - remove_old_images $REPO echo -e "${BOLD}Tagged${RESET}: aztecprotocol/$REPO:latest" diff --git a/build-system/scripts/can_use_ci_cache b/build-system/scripts/can_use_ci_cache new file mode 100755 index 00000000000..2831dabb2bb --- /dev/null +++ b/build-system/scripts/can_use_ci_cache @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +set -euo pipefail +type docker &> /dev/null && docker ps 2>&1 > /dev/null && [ -f ~/.aws/credentials ] && ecr_login > /dev/null || exit 1 \ No newline at end of file diff --git a/build-system/scripts/cond_run_compose b/build-system/scripts/cond_run_compose index 89c8749690b..b67fa075e06 100755 --- a/build-system/scripts/cond_run_compose +++ b/build-system/scripts/cond_run_compose @@ -27,7 +27,6 @@ if ! check_rebuild $SUCCESS_TAG $REPOSITORY; then echo "Pulling $REPO..." REPO_IMAGE_URI=$(calculate_image_uri $REPO) retry docker pull $REPO_IMAGE_URI - docker tag $REPO_IMAGE_URI $ECR_DEPLOY_URL/$REPO docker tag $REPO_IMAGE_URI aztecprotocol/$REPO done diff --git a/build-system/scripts/deploy_ecr b/build-system/scripts/deploy_ecr deleted file mode 100755 index c156f99e71e..00000000000 --- a/build-system/scripts/deploy_ecr +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash -[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace -set -eu - -REPOSITORY=$1 -ARCH=${2:-""} -IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY) -if [[ -n "$ARCH" ]]; then - IMAGE_COMMIT_URI=$IMAGE_COMMIT_URI-$ARCH -fi - -# Login to build region and pull the build. -retry ensure_repo $REPOSITORY $ECR_REGION -retry docker pull $IMAGE_COMMIT_URI > /dev/null 2>&1 - -# Ensure ECR repository exists in deployment region. -retry ensure_repo $REPOSITORY $ECR_DEPLOY_REGION - -# Push image to deployment repo with commit hash tag and optional architecture e.g: -# falafel:deadbeefcafebabe1337c0de-x86_64 -IMAGE_DEPLOY_COMMIT_URI=$ECR_DEPLOY_URL/$REPOSITORY:$COMMIT_HASH -CURRENT_TAG=$COMMIT_HASH -NEW_TAG_COMMIT_TAG=$PROJECT-$COMMIT_TAG -NEW_TAG_DEPLOY_TAG=$DEPLOY_TAG -if [[ -n "$ARCH" ]]; then - IMAGE_DEPLOY_COMMIT_URI=$IMAGE_DEPLOY_COMMIT_URI-$ARCH - CURRENT_TAG=$CURRENT_TAG-$ARCH - NEW_TAG_COMMIT_TAG=$NEW_TAG_COMMIT_TAG-$ARCH - NEW_TAG_DEPLOY_TAG=$NEW_TAG_DEPLOY_TAG-$ARCH -fi -retry docker tag $IMAGE_COMMIT_URI $IMAGE_DEPLOY_COMMIT_URI -retry docker push $IMAGE_DEPLOY_COMMIT_URI - -# Tag image with full version if we have one. Allows deployment of precise image version if rollback needed. -if [ -n "${COMMIT_TAG:-}" ]; then - retry tag_remote_image $REPOSITORY $CURRENT_TAG $NEW_TAG_COMMIT_TAG $ECR_DEPLOY_REGION -fi - -retry tag_remote_image $REPOSITORY $CURRENT_TAG $NEW_TAG_DEPLOY_TAG $ECR_DEPLOY_REGION diff --git a/build-system/scripts/generate_circleci_config.py b/build-system/scripts/generate_circleci_config.py index f04328646d0..51dafde0af1 100755 --- a/build-system/scripts/generate_circleci_config.py +++ b/build-system/scripts/generate_circleci_config.py @@ -20,7 +20,7 @@ def get_manifest_job_names(): def is_already_built_circleci_job(circleci_job, already_built_manifest_jobs): """ - This function checks if a given CircleCI job is associated with a specific already-built manifest job. + This function checks if a given CircleCI job is associated with a specific already-built manifest job. It does so by checking the job's steps for an 'aztec_manifest_key' that contain references to manifest names. We want to see at least one such key, and for all such keys to be in 'already_built_manifest_jobs'. """ @@ -43,7 +43,7 @@ def is_already_built_circleci_job(circleci_job, already_built_manifest_jobs): return False matching_steps += 1 # All steps have matched - but make sure that's actually more than one step - return matching_steps > 0 + return matching_steps > 0 def get_already_built_circleci_job_names(circleci_jobs): already_built_manifest_jobs = list(get_already_built_manifest_job_names()) @@ -75,11 +75,11 @@ def get_already_built_manifest_job_names(): def remove_jobs_from_workflow(jobs, to_remove): """ Removes jobs from a given CircleCI JSON workflow. - + Parameters: jobs (dict): The JSON object representing the CircleCI workflow jobs dependencies portion. to_remove (list): The list of jobs to be removed from the workflow. - + Returns: dict: The new JSON object with specified jobs removed. """ @@ -97,7 +97,7 @@ def remove_jobs_from_workflow(jobs, to_remove): if __name__ == '__main__': # The CircleCI workflow as a JSON string (Replace this with your actual workflow) - + # Convert the JSON string to a Python dictionary workflow_dict = yaml.safe_load(open('.circleci/config.yml')) diff --git a/build-system/scripts/remote_run_script b/build-system/scripts/remote_run_script index 059417cced0..855b3f14610 100755 --- a/build-system/scripts/remote_run_script +++ b/build-system/scripts/remote_run_script @@ -28,11 +28,10 @@ ssh -A -F $SSH_CONFIG_PATH $IP " export BRANCH=${BRANCH:-} export PULL_REQUEST=${PULL_REQUEST:-} export DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD - export ECR_DEPLOY_URL=$ECR_DEPLOY_URL export ECR_URL=$ECR_URL export BUILD_SYSTEM_DEBUG=${BUILD_SYSTEM_DEBUG:-} - # temp while we transitioning to avm + # temp while we transitioning to avm export AVM_ENABLED=${AVM_ENABLED:-} ./remote_runner $@ " diff --git a/build-system/scripts/remove_old_images b/build-system/scripts/remove_old_images index 7c1c927e6b6..eb8e4c6f046 100755 --- a/build-system/scripts/remove_old_images +++ b/build-system/scripts/remove_old_images @@ -7,7 +7,7 @@ REPOSITORY=$1 shift IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY) -for IMAGE in $(docker images --format "{{.ID}}" $ECR_URL/$REPOSITORY --filter "before=$IMAGE_COMMIT_URI"); do +for IMAGE in $(docker images --format "{{.ID}}" aztecprotocol/$REPOSITORY --filter "before=$IMAGE_COMMIT_URI"); do echo "Removing $IMAGE..." docker rmi --force $IMAGE done diff --git a/build-system/scripts/request_spot b/build-system/scripts/request_spot index 6c4e1d1bb2d..93c5d9a990b 100755 --- a/build-system/scripts/request_spot +++ b/build-system/scripts/request_spot @@ -32,10 +32,10 @@ fi # Construct the full instance type. We use r6a currently. if [ "$ARCH" == "x86_64" ]; then MACHINE_TYPE="r6a" - AMI="ami-0e5df77ac318c7a18" + AMI="ami-04d8422a9ba4de80f" elif [ "$ARCH" == "arm64" ]; then MACHINE_TYPE="r7g" - AMI="ami-02c5a5064fe72b22e" + AMI="ami-0d8a9b0419ddb331a" else echo "Unknown arch: $ARCH" exit 1 diff --git a/build-system/scripts/setup_env b/build-system/scripts/setup_env index 14482b99fb0..11fd7e70bfa 100755 --- a/build-system/scripts/setup_env +++ b/build-system/scripts/setup_env @@ -87,7 +87,6 @@ echo export ECR_REGION=us-east-2 >> $BASH_ENV echo export AWS_ACCOUNT=278380418400 >> $BASH_ENV echo export ECR_URL=278380418400.dkr.ecr.us-east-2.amazonaws.com >> $BASH_ENV echo export ECR_DEPLOY_REGION=eu-west-2 >> $BASH_ENV -echo export ECR_DEPLOY_URL=278380418400.dkr.ecr.eu-west-2.amazonaws.com >> $BASH_ENV echo export DOCKERHUB_ACCOUNT=aztecprotocol >> $BASH_ENV echo export DOCKERHUB_USERNAME=aztecprotocolci >> $BASH_ENV echo export PROJECT=$PROJECT >> $BASH_ENV diff --git a/docs/Dockerfile b/docs/Dockerfile index 847379f1ccf..d963bf86ec7 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -1,4 +1,4 @@ -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project +FROM aztecprotocol/yarn-project WORKDIR /usr/src COPY . . WORKDIR /usr/src/docs diff --git a/l1-contracts/bootstrap.sh b/l1-contracts/bootstrap.sh index 489ee2a80eb..fa8189b567d 100755 --- a/l1-contracts/bootstrap.sh +++ b/l1-contracts/bootstrap.sh @@ -19,7 +19,7 @@ fi . ./scripts/install_foundry.sh # Attempt to just pull artefacts from CI and exit on success. -./bootstrap_cache.sh && exit +[ -n "${USE_CACHE:-}" ] && ./bootstrap_cache.sh && exit # Clean rm -rf broadcast cache out serve diff --git a/l1-contracts/bootstrap_cache.sh b/l1-contracts/bootstrap_cache.sh index 39ac1ecfc7a..911796ce9b5 100755 --- a/l1-contracts/bootstrap_cache.sh +++ b/l1-contracts/bootstrap_cache.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash set -eu -[ -z "${NO_CACHE:-}" ] && type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 - cd "$(dirname "$0")" source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null diff --git a/noir-projects/Dockerfile b/noir-projects/Dockerfile index 2e2f48e4de8..546d01e634a 100644 --- a/noir-projects/Dockerfile +++ b/noir-projects/Dockerfile @@ -1,5 +1,5 @@ -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir as noir -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/avm-transpiler as transpiler +FROM aztecprotocol/noir as noir +FROM aztecprotocol/avm-transpiler as transpiler FROM ubuntu:lunar AS builder # Copy in nargo diff --git a/noir-projects/bootstrap.sh b/noir-projects/bootstrap.sh index 2dc019310b0..319d28ae211 100755 --- a/noir-projects/bootstrap.sh +++ b/noir-projects/bootstrap.sh @@ -16,7 +16,7 @@ if [ -n "$CMD" ]; then fi # Attempt to just pull artefacts from CI and exit on success. -./bootstrap_cache.sh && exit +[ -n "${USE_CACHE:-}" ] && ./bootstrap_cache.sh && exit PROJECTS=( noir-contracts diff --git a/noir-projects/bootstrap_cache.sh b/noir-projects/bootstrap_cache.sh index 1f0500b21ab..898fbdbea17 100755 --- a/noir-projects/bootstrap_cache.sh +++ b/noir-projects/bootstrap_cache.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash set -eu -[ -z "${NO_CACHE:-}" ] && type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 - cd "$(dirname "$0")" source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null diff --git a/noir/bootstrap.sh b/noir/bootstrap.sh index 1f9506904a4..b4373eb9a80 100755 --- a/noir/bootstrap.sh +++ b/noir/bootstrap.sh @@ -16,7 +16,7 @@ if [ -n "$CMD" ]; then fi # Attempt to just pull artefacts from CI and exit on success. -./bootstrap_cache.sh && exit +[ -n "${USE_CACHE:-}" ] && ./bootstrap_cache.sh && exit ./scripts/bootstrap_native.sh ./scripts/bootstrap_packages.sh \ No newline at end of file diff --git a/noir/bootstrap_cache.sh b/noir/bootstrap_cache.sh index d06aa493662..1cec6c81d8e 100755 --- a/noir/bootstrap_cache.sh +++ b/noir/bootstrap_cache.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash set -eu -[ -z "${NO_CACHE:-}" ] && type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 - cd "$(dirname "$0")" source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null diff --git a/yarn-project/Dockerfile b/yarn-project/Dockerfile index aea67d0c588..3349532fa20 100644 --- a/yarn-project/Dockerfile +++ b/yarn-project/Dockerfile @@ -4,11 +4,11 @@ # - Run the tests. # - Run the formatter checks. # Any subsequent build steps needed to support downstream containers should be done in those containers build files. -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/l1-contracts as contracts -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-projects as noir-projects -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/boxes-files as boxes-files +FROM aztecprotocol/l1-contracts as contracts +FROM aztecprotocol/noir-projects as noir-projects +FROM aztecprotocol/boxes-files as boxes-files -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-base +FROM aztecprotocol/yarn-project-base COPY --from=contracts /usr/src/l1-contracts /usr/src/l1-contracts COPY --from=noir-projects /usr/src/noir-projects /usr/src/noir-projects COPY --from=boxes-files /usr/src/boxes /usr/src/boxes diff --git a/yarn-project/Dockerfile.prod b/yarn-project/Dockerfile.prod index b935d818fd6..c355f7b6a1f 100644 --- a/yarn-project/Dockerfile.prod +++ b/yarn-project/Dockerfile.prod @@ -1,6 +1,6 @@ # This productionifies the workspace, removing all developer dependencies and producing a final slim image from which # we then generate downstream multiarch containers to execute the specific projects. -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project AS yarn-project +FROM aztecprotocol/yarn-project AS yarn-project # Need new arch specific image. FROM node:18.19.0 AS builder diff --git a/yarn-project/aztec-faucet/Dockerfile b/yarn-project/aztec-faucet/Dockerfile index f4567db7482..047f7cd0cac 100644 --- a/yarn-project/aztec-faucet/Dockerfile +++ b/yarn-project/aztec-faucet/Dockerfile @@ -1,4 +1,4 @@ -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-prod +FROM aztecprotocol/yarn-project-prod WORKDIR /usr/src/yarn-project/aztec-faucet ENTRYPOINT ["node", "--no-warnings", "/usr/src/yarn-project/aztec-faucet/dest/bin/index.js"] EXPOSE 8080 diff --git a/yarn-project/aztec/Dockerfile b/yarn-project/aztec/Dockerfile index e8a8e8d45d1..13b2000744d 100644 --- a/yarn-project/aztec/Dockerfile +++ b/yarn-project/aztec/Dockerfile @@ -1,4 +1,4 @@ -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-prod AS yarn-project-prod +FROM aztecprotocol/yarn-project-prod AS yarn-project-prod ENTRYPOINT ["node", "--no-warnings", "/usr/src/yarn-project/aztec/dest/bin/index.js"] EXPOSE 8080 diff --git a/yarn-project/cli/Dockerfile b/yarn-project/cli/Dockerfile index 21dfcc65e97..b782d46e801 100644 --- a/yarn-project/cli/Dockerfile +++ b/yarn-project/cli/Dockerfile @@ -1,4 +1,4 @@ -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-prod AS yarn-project-prod +FROM aztecprotocol/yarn-project-prod AS yarn-project-prod ENTRYPOINT ["node", "--no-warnings", "/usr/src/yarn-project/cli/dest/bin/index.js"] # The version has been updated in yarn-project-prod. diff --git a/yarn-project/end-to-end/Dockerfile b/yarn-project/end-to-end/Dockerfile index 3e81baa6f2c..49f5a14711d 100644 --- a/yarn-project/end-to-end/Dockerfile +++ b/yarn-project/end-to-end/Dockerfile @@ -1,4 +1,4 @@ -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project AS builder +FROM aztecprotocol/yarn-project AS builder # Build web bundle for browser tests WORKDIR /usr/src/yarn-project/end-to-end diff --git a/yarn-project/p2p-bootstrap/scripts/docker-compose-bootstrap.yml b/yarn-project/p2p-bootstrap/scripts/docker-compose-bootstrap.yml index 9ea2293cb24..d9788104ec6 100644 --- a/yarn-project/p2p-bootstrap/scripts/docker-compose-bootstrap.yml +++ b/yarn-project/p2p-bootstrap/scripts/docker-compose-bootstrap.yml @@ -1,7 +1,7 @@ version: '3' services: p2p-bootstrap: - image: 278380418400.dkr.ecr.eu-west-2.amazonaws.com/aztec:latest + image: aztecprotocol/aztec:latest command: 'start --p2p-bootstrap' ports: - '40400:40400' diff --git a/yarn-project/yarn-project-base/Dockerfile b/yarn-project/yarn-project-base/Dockerfile index b53ed652bc0..62c4b55fb42 100644 --- a/yarn-project/yarn-project-base/Dockerfile +++ b/yarn-project/yarn-project-base/Dockerfile @@ -42,8 +42,8 @@ # RUN yarn workspaces focus --production && yarn cache clean && rm -rf ../**/src # - Create final slim image by copying needed dirs into a fresh image. # -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/bb.js as bb.js -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-packages as noir-packages +FROM aztecprotocol/bb.js as bb.js +FROM aztecprotocol/noir-packages as noir-packages FROM node:18.19.0 RUN apt update && apt install -y jq curl perl && rm -rf /var/lib/apt/lists/* && apt-get clean diff --git a/yarn-project/yarn-project-base/Dockerfile.dockerignore.v24 b/yarn-project/yarn-project-base/Dockerfile.dockerignore.v24 index 421235079e3..0627db619ac 100644 --- a/yarn-project/yarn-project-base/Dockerfile.dockerignore.v24 +++ b/yarn-project/yarn-project-base/Dockerfile.dockerignore.v24 @@ -5,7 +5,7 @@ # # This took a fair bit of trial and error to get the right result. If you need to meddle, try: # - Commenting out everything after the COPY . . in the dockerfile. -# - ONLY_TARGET=1 ../bootstrap_docker.sh yarn-project-base && docker run -ti --rm 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-base:latest sh -c 'du -ha .' +# - ONLY_TARGET=1 ../bootstrap_docker.sh yarn-project-base && docker run -ti --rm aztecprotocol/yarn-project-base:latest sh -c 'du -ha .' # Exclude everything to start. *