From 8e75fe5c47250e860a4eae4dbf0973c503221720 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 15 Mar 2024 18:54:33 -0400 Subject: [PATCH] feat: initial Earthly CI (#5069) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces earthly as an alternative CI that hopes to eventually replace our current build-system. https://docs.earthly.dev/ is a build system that combines Makefiles and Dockerfiles. This is basically exactly what our system needed, IMO, and has some nice things figured out. Hope is to reduce complexity of working with the build system by a good chunk. Core changes: - we have a github actions CI that runs a single end to end test inside earthly for arm64 and x86_64 - new Earthfile's now mirror the Dockerfile's, notable differences: - we build our own foundry package for ARM support - we build our own wasi-sdk package for ARM support - grumpkin SRS is no longer generated on the spot, but downloaded like bn254 SRS - we don't inject any commit hashes for Noir as this would cause spurious rebuilds as any difference stops caching, instead we inject a content hash (to be revisited) Side changes: - since we build our own wasi-sdk 21 package, and it is clang18, some compilation workarounds - allow specifying a different nargo and acvm binary in build - small output tweaks --------- Co-authored-by: Charlie Lye Co-authored-by: Innokentii Sennovskii Co-authored-by: Cody Gunton Co-authored-by: Alex Gherghisan Co-authored-by: Mitchell Tracy Co-authored-by: Jan Beneš Co-authored-by: esau <152162806+sklppy88@users.noreply.github.com> Co-authored-by: Facundo Co-authored-by: josh crites Co-authored-by: Tom French <15848336+TomAFrench@users.noreply.github.com> Co-authored-by: Álvaro Rodríguez Co-authored-by: Ilyas Ridhuan --- .circleci/config.yml | 5 +- .github/workflows/ci.yml | 46 +++++ .../workflows/protocol-circuits-gate-diff.yml | 4 + .gitignore | 3 +- Earthfile | 31 +++ README.md | 18 +- avm-transpiler/Earthfile | 23 +++ barretenberg/README.md | 2 - barretenberg/cpp/.aztec-packages-commit | 1 - barretenberg/cpp/CMakePresets.json | 11 +- barretenberg/cpp/Earthfile | 176 ++++++++++++++++++ .../scripts/benchmark_field_ops_percentage.sh | 4 +- barretenberg/cpp/scripts/benchmark_wasm.sh | 2 +- .../scripts/compute_field_operations_time.py | 2 +- barretenberg/cpp/scripts/install-wasi-sdk.sh | 5 +- barretenberg/cpp/src/CMakeLists.txt | 6 +- .../cpp/src/barretenberg/barretenberg.hpp | 74 -------- .../permutation_widget_impl.hpp | 2 +- .../polynomials/polynomial_arithmetic.cpp | 27 ++- .../polynomials/polynomial_arithmetic.hpp | 9 +- barretenberg/cpp/srs_db/Earthfile | 19 ++ barretenberg/cpp/srs_db/download_grumpkin.sh | 13 +- barretenberg/cpp/srs_db/download_ignition.sh | 65 +------ barretenberg/cpp/srs_db/download_srs.sh | 52 ++++++ .../cpp/srs_db/grumpkin/monomial/README.md | 16 -- .../cpp/srs_db/ignition/monomial/checksums | 21 --- .../cpp/srs_db/ignition/monomial/g2.dat | 1 - barretenberg/ts/.earthlyignore | 1 + barretenberg/ts/Earthfile | 48 +++++ boxes/Earthfile | 18 ++ .../{remote_runner => remote_initialize} | 2 - build-system/scripts/remote_run_script | 9 +- build-system/scripts/setup_env | 1 + foundry/Dockerfile | 35 ++++ foundry/Earthfile | 11 ++ l1-contracts/Earthfile | 23 +++ noir-projects/.earthlyignore | 1 + noir-projects/Earthfile | 24 +++ noir-projects/noir-contracts/bootstrap.sh | 6 +- .../noir-protocol-circuits/bootstrap.sh | 3 +- noir/Earthfile | 111 +++++++++++ .../.github/scripts/wasm-bindgen-install.sh | 2 +- scripts/earthly | 31 +++ scripts/earthly-config.yml | 3 + scripts/earthly-timed | 55 ++++++ yarn-project/.earthlyignore | 56 ++++++ yarn-project/.gitignore | 49 ++++- yarn-project/Earthfile | 114 ++++++++++++ yarn-project/accounts/.gitignore | 1 - yarn-project/aztec-faucet/.gitignore | 1 - yarn-project/aztec-node/.gitignore | 1 - yarn-project/aztec.js/.gitignore | 1 - yarn-project/aztec/.gitignore | 1 - yarn-project/circuits.js/.gitignore | 1 - yarn-project/docs/.gitignore | 1 - yarn-project/end-to-end/.gitignore | 5 - yarn-project/end-to-end/Earthfile | 151 +++++++++++++++ .../end-to-end/scripts/docker-compose.yml | 9 +- yarn-project/entrypoints/.gitignore | 1 - yarn-project/l1-artifacts/.gitignore | 1 - yarn-project/noir-compiler/.gitignore | 4 - yarn-project/noir-contracts.js/.gitignore | 4 - .../noir-protocol-circuits-types/.gitignore | 6 - yarn-project/package.json | 2 +- yarn-project/protocol-contracts/.gitignore | 1 - yarn-project/scripts/.gitignore | 1 - yarn-project/scripts/version_packages.sh | 2 + 67 files changed, 1159 insertions(+), 276 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 Earthfile create mode 100644 avm-transpiler/Earthfile delete mode 100644 barretenberg/cpp/.aztec-packages-commit create mode 100644 barretenberg/cpp/Earthfile delete mode 100644 barretenberg/cpp/src/barretenberg/barretenberg.hpp create mode 100644 barretenberg/cpp/srs_db/Earthfile create mode 100755 barretenberg/cpp/srs_db/download_srs.sh delete mode 100644 barretenberg/cpp/srs_db/grumpkin/monomial/README.md delete mode 100644 barretenberg/cpp/srs_db/ignition/monomial/checksums delete mode 100644 barretenberg/cpp/srs_db/ignition/monomial/g2.dat create mode 100644 barretenberg/ts/.earthlyignore create mode 100644 barretenberg/ts/Earthfile create mode 100644 boxes/Earthfile rename build-system/scripts/{remote_runner => remote_initialize} (96%) create mode 100644 foundry/Dockerfile create mode 100644 foundry/Earthfile create mode 100644 l1-contracts/Earthfile create mode 100644 noir-projects/.earthlyignore create mode 100644 noir-projects/Earthfile create mode 100644 noir/Earthfile create mode 100755 scripts/earthly create mode 100644 scripts/earthly-config.yml create mode 100755 scripts/earthly-timed create mode 100644 yarn-project/.earthlyignore create mode 100644 yarn-project/Earthfile delete mode 100644 yarn-project/accounts/.gitignore delete mode 100644 yarn-project/aztec-faucet/.gitignore delete mode 100644 yarn-project/aztec-node/.gitignore delete mode 100644 yarn-project/aztec.js/.gitignore delete mode 100644 yarn-project/aztec/.gitignore delete mode 100644 yarn-project/circuits.js/.gitignore delete mode 100644 yarn-project/docs/.gitignore delete mode 100644 yarn-project/end-to-end/.gitignore create mode 100644 yarn-project/end-to-end/Earthfile delete mode 100644 yarn-project/entrypoints/.gitignore delete mode 100644 yarn-project/l1-artifacts/.gitignore delete mode 100644 yarn-project/noir-compiler/.gitignore delete mode 100644 yarn-project/noir-contracts.js/.gitignore delete mode 100644 yarn-project/noir-protocol-circuits-types/.gitignore delete mode 100644 yarn-project/protocol-contracts/.gitignore delete mode 100644 yarn-project/scripts/.gitignore diff --git a/.circleci/config.yml b/.circleci/config.yml index 913e0cd6bce..1a69da686a0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1313,7 +1313,10 @@ workflows: - barretenberg-stdlib-tests: *bb_test - barretenberg-stdlib-recursion-ultra-tests: *bb_test - barretenberg-acir-tests-bb: *bb_acir_tests - - barretenberg-acir-tests-bb-sol: *bb_acir_tests + - barretenberg-acir-tests-bb-sol: + requires: + - barretenberg-x86_64-linux-clang-sol + <<: *bb_acir_tests - barretenberg-docs: *defaults - bb-js: requires: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000000..de83032b8fd --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,46 @@ +name: Run CI with Earthly +on: + push: + branches: + - master + pull_request: {} + workflow_dispatch: {} + +jobs: + ci: + runs-on: ubuntu-latest + # run ci for both x86_64 and arm64 + strategy: {matrix: {environment: [x86, arm]}} + # cancel if reran on same PR if exists, otherwise if on same commit + concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ matrix.environment }} + cancel-in-progress: true + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + submodules: recursive + + - name: Setup + run: | + mkdir -p ~/.ssh + echo DOCKER_HOST=ssh://build-instance-${{ matrix.environment }}.aztecprotocol.com >> $GITHUB_ENV + echo ${{ secrets.DOCKERHUB_PASSWORD}} | docker login -u aztecprotocolci --password-stdin + echo ${{ secrets.BUILD_INSTANCE_SSH_KEY }} | base64 -d > ~/.ssh/build_instance_key + chmod 600 ~/.ssh/build_instance_key + cat > ~/.ssh/config < $ $ diff --git a/barretenberg/cpp/src/barretenberg/barretenberg.hpp b/barretenberg/cpp/src/barretenberg/barretenberg.hpp deleted file mode 100644 index 659f3f2d115..00000000000 --- a/barretenberg/cpp/src/barretenberg/barretenberg.hpp +++ /dev/null @@ -1,74 +0,0 @@ -#pragma once - -// External Barretenberg C++ API -#include "common/bbmalloc.hpp" -#include "common/container.hpp" -#include "common/map.hpp" -#include "common/mem.hpp" -#include "common/serialize.hpp" -#include "common/streams.hpp" -#include "common/throw_or_abort.hpp" -#include "crypto/blake2s/blake2s.hpp" -#include "crypto/blake3s/blake3s.hpp" -#include "crypto/ecdsa/ecdsa.hpp" -#include "crypto/generators/generator_data.hpp" -#include "crypto/keccak/keccak.hpp" -#include "crypto/pedersen_commitment/pedersen.hpp" -#include "crypto/pedersen_hash/pedersen.hpp" -#include "crypto/poseidon2/poseidon2.hpp" -#include "crypto/schnorr/schnorr.hpp" -#include "crypto/sha256/sha256.hpp" -#include "ecc/curves/bn254/fq.hpp" -#include "ecc/curves/bn254/fr.hpp" -#include "ecc/curves/bn254/g1.hpp" -#include "ecc/curves/grumpkin/grumpkin.hpp" -#include "numeric/random/engine.hpp" -#include "numeric/uint256/uint256.hpp" -#include "plonk/proof_system/types/proof.hpp" -#include "plonk/proof_system/verification_key/verification_key.hpp" -#include "proof_system/circuit_builder/ultra_circuit_builder.hpp" -#include "proof_system/types/circuit_type.hpp" -// TODO(https://github.com/AztecProtocol/barretenberg/issues/491): -// consider helper header(s) for serialization and other non-stdlib includes -// - possibly: common, serialize, srs, transcript -#include "serialize/cbind.hpp" -#include "serialize/cbind_fwd.hpp" -#include "serialize/msgpack.hpp" -#include "serialize/test_helper.hpp" -#include "srs/global_crs.hpp" -#include "stdlib/commitment/pedersen/pedersen.hpp" -#include "stdlib/encryption/ecdsa/ecdsa.hpp" -#include "stdlib/encryption/schnorr/schnorr.hpp" -#include "stdlib/hash/blake2s/blake2s.hpp" -#include "stdlib/hash/blake3s/blake3s.hpp" -#include "stdlib/hash/pedersen/pedersen.hpp" -#include "stdlib/hash/poseidon2/poseidon2.hpp" -#include "stdlib/merkle_tree/hash.hpp" -#include "stdlib/merkle_tree/membership.hpp" -#include "stdlib/merkle_tree/memory_store.hpp" -#include "stdlib/merkle_tree/memory_tree.hpp" -#include "stdlib/merkle_tree/merkle_tree.hpp" -// TODO(https://github.com/AztecProtocol/aztec-packages/issues/728): -// Consider moving nullifier tree logic out of barretenberg into aztec repo -#include "barretenberg/plonk/transcript/manifest.hpp" -#include "stdlib/merkle_tree/nullifier_tree/nullifier_leaf.hpp" -#include "stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp" -#include "stdlib/merkle_tree/nullifier_tree/nullifier_tree.hpp" -#include "stdlib/plonk_recursion/aggregation_state/aggregation_state.hpp" -#include "stdlib/plonk_recursion/aggregation_state/native_aggregation_state.hpp" -#include "stdlib/plonk_recursion/verification_key/verification_key.hpp" -#include "stdlib/plonk_recursion/verifier/program_settings.hpp" -#include "stdlib/plonk_recursion/verifier/verifier.hpp" -#include "stdlib/primitives/address/address.hpp" -#include "stdlib/primitives/bigfield/bigfield.hpp" -#include "stdlib/primitives/biggroup/biggroup.hpp" -#include "stdlib/primitives/bit_array/bit_array.hpp" -#include "stdlib/primitives/bool/bool.hpp" -#include "stdlib/primitives/byte_array/byte_array.hpp" -#include "stdlib/primitives/curves/bn254.hpp" -#include "stdlib/primitives/field/array.hpp" -#include "stdlib/primitives/field/field.hpp" -#include "stdlib/primitives/group/cycle_group.hpp" -#include "stdlib/primitives/packed_byte_array/packed_byte_array.hpp" -#include "stdlib/primitives/uint/uint.hpp" -#include "stdlib/primitives/witness/witness.hpp" diff --git a/barretenberg/cpp/src/barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget_impl.hpp b/barretenberg/cpp/src/barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget_impl.hpp index 50ae2dccefa..387ed72dc56 100644 --- a/barretenberg/cpp/src/barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget_impl.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget_impl.hpp @@ -66,7 +66,7 @@ void ProverPermutationWidget accumulators_ptrs[num_accumulators]; fr* accumulators[num_accumulators]; // Allocate the required number of length n scratch space arrays diff --git a/barretenberg/cpp/src/barretenberg/polynomials/polynomial_arithmetic.cpp b/barretenberg/cpp/src/barretenberg/polynomials/polynomial_arithmetic.cpp index 3530b816f3b..0694349f95d 100644 --- a/barretenberg/cpp/src/barretenberg/polynomials/polynomial_arithmetic.cpp +++ b/barretenberg/cpp/src/barretenberg/polynomials/polynomial_arithmetic.cpp @@ -1120,38 +1120,36 @@ bb::polynomial_arithmetic::LagrangeEvaluations get_lagrange_evaluations( // L_i(z) = L_1(ʓ.ω^{1-i}) = ------------------ // n.(ʓ.ω^{1-i)} - 1) // -template - requires SupportsFFT -Fr compute_barycentric_evaluation(const Fr* coeffs, +fr compute_barycentric_evaluation(const fr* coeffs, const size_t num_coeffs, - const Fr& z, - const EvaluationDomain& domain) + const fr& z, + const EvaluationDomain& domain) { - Fr* denominators = static_cast(aligned_alloc(64, sizeof(Fr) * num_coeffs)); + fr* denominators = static_cast(aligned_alloc(64, sizeof(fr) * num_coeffs)); - Fr numerator = z; + fr numerator = z; for (size_t i = 0; i < domain.log2_size; ++i) { numerator.self_sqr(); } - numerator -= Fr::one(); + numerator -= fr::one(); numerator *= domain.domain_inverse; // (ʓ^n - 1) / n - denominators[0] = z - Fr::one(); - Fr work_root = domain.root_inverse; // ω^{-1} + denominators[0] = z - fr::one(); + fr work_root = domain.root_inverse; // ω^{-1} for (size_t i = 1; i < num_coeffs; ++i) { denominators[i] = work_root * z; // denominators[i] will correspond to L_[i+1] (since our 'commented maths' notation indexes // L_i from 1). So ʓ.ω^{-i} = ʓ.ω^{1-(i+1)} is correct for L_{i+1}. - denominators[i] -= Fr::one(); // ʓ.ω^{-i} - 1 + denominators[i] -= fr::one(); // ʓ.ω^{-i} - 1 work_root *= domain.root_inverse; } - Fr::batch_invert(denominators, num_coeffs); + fr::batch_invert(denominators, num_coeffs); - Fr result = Fr::zero(); + fr result = fr::zero(); for (size_t i = 0; i < num_coeffs; ++i) { - Fr temp = coeffs[i] * denominators[i]; // f_i * 1/(ʓ.ω^{-i} - 1) + fr temp = coeffs[i] * denominators[i]; // f_i * 1/(ʓ.ω^{-i} - 1) result = result + temp; } @@ -1401,7 +1399,6 @@ template void divide_by_pseudo_vanishing_polynomial(std::vector, const size_t); template fr compute_kate_opening_coefficients(const fr*, fr*, const fr&, const size_t); template LagrangeEvaluations get_lagrange_evaluations(const fr&, const EvaluationDomain&, const size_t); -template fr compute_barycentric_evaluation(const fr*, const size_t, const fr&, const EvaluationDomain&); template void compress_fft(const fr*, fr*, const size_t, const size_t); template fr evaluate_from_fft(const fr*, const EvaluationDomain&, const fr&, const EvaluationDomain&); template fr compute_sum(const fr*, const size_t); diff --git a/barretenberg/cpp/src/barretenberg/polynomials/polynomial_arithmetic.hpp b/barretenberg/cpp/src/barretenberg/polynomials/polynomial_arithmetic.hpp index 61ee093e698..4de39d5a918 100644 --- a/barretenberg/cpp/src/barretenberg/polynomials/polynomial_arithmetic.hpp +++ b/barretenberg/cpp/src/barretenberg/polynomials/polynomial_arithmetic.hpp @@ -156,11 +156,10 @@ template LagrangeEvaluations get_lagrange_evaluations(const Fr& z, const EvaluationDomain& domain, const size_t num_roots_cut_out_of_vanishing_polynomial = 4); -template -Fr compute_barycentric_evaluation(const Fr* coeffs, - const size_t num_coeffs, - const Fr& z, - const EvaluationDomain& domain); +fr compute_barycentric_evaluation(const fr* coeffs, + unsigned long num_coeffs, + const fr& z, + const EvaluationDomain& domain); // Convert an fft with `current_size` point evaluations, to one with `current_size >> compress_factor` point evaluations template requires SupportsFFT diff --git a/barretenberg/cpp/srs_db/Earthfile b/barretenberg/cpp/srs_db/Earthfile new file mode 100644 index 00000000000..36688dac614 --- /dev/null +++ b/barretenberg/cpp/srs_db/Earthfile @@ -0,0 +1,19 @@ +VERSION 0.8 +FROM ubuntu:lunar + +RUN apt-get update && apt-get install -y curl + +build: + WORKDIR /build + COPY ./*.sh . + RUN ./download_ignition.sh 3 + RUN ./download_grumpkin.sh + # export srs-db for runners + SAVE ARTIFACT ignition ignition + SAVE ARTIFACT ignition grumpkin + +build-local: + # copy files locally + FROM +download + SAVE ARTIFACT ignition AS LOCAL ignition + SAVE ARTIFACT grumpkin AS LOCAL grumpkin diff --git a/barretenberg/cpp/srs_db/download_grumpkin.sh b/barretenberg/cpp/srs_db/download_grumpkin.sh index fb59a1ec806..56297198d27 100755 --- a/barretenberg/cpp/srs_db/download_grumpkin.sh +++ b/barretenberg/cpp/srs_db/download_grumpkin.sh @@ -1,11 +1,8 @@ #!/bin/sh -# TODO(https://github.com/AztecProtocol/barretenberg/issues/813) We don't *actually* download grumpkin yet. -# this just generates grumpkin points and links in a place where run_acir_tests.sh expects it. -# The above issue tracks the final pieces here. +# TODO(https://github.com/AztecProtocol/barretenberg/issues/898): Grumpkin needs to match new layout. set -eu - -# Enter build directory sibling to our script folder. -cd $(dirname $0)/../build -./bin/grumpkin_srs_gen 1048576 -mkdir -p ~/.bb-crs +# Enter script directory. +cd $(dirname $0) +./download_srs.sh "TEST%20GRUMPKIN" grumpkin/monomial 1 $@ +mkdir -p ~/.bb-crs ln -s ../srs_db/grumpkin/monomial ~/.bb-crs/monomial \ No newline at end of file diff --git a/barretenberg/cpp/srs_db/download_ignition.sh b/barretenberg/cpp/srs_db/download_ignition.sh index 4b0df0ee9fc..4e469d7505f 100755 --- a/barretenberg/cpp/srs_db/download_ignition.sh +++ b/barretenberg/cpp/srs_db/download_ignition.sh @@ -1,68 +1,5 @@ #!/bin/sh -# Downloads the ignition trusted setup transcripts. -# -# See here for details of the contents of the transcript.dat files: -# https://github.com/AztecProtocol/ignition-verification/blob/master/Transcript_spec.md -# -# To download all transcripts. -# ./download_ignition.sh -# -# To download a range of transcripts, e.g. 0, 1 and 2. -# ./download_ignition.sh 2 -# -# If a checksums file is available, it will be used to validate if a download is required -# and also check the validity of the downloaded transcripts. If not the script downloads -# whatever is requested but does not check the validity of the downloads. set -eu - # Enter script directory. cd $(dirname $0) - -mkdir -p ignition -cd ignition -mkdir -p monomial -cd monomial -NUM=${1:-19} -RANGE_START=${2:-} -RANGE_END=${3:-} -APPEND=${4:-"false"} - -if command -v sha256sum > /dev/null; then - SHASUM=sha256sum -else - SHASUM="shasum -a 256" -fi - -checksum() { - grep transcript${1}.dat checksums | $SHASUM -c - return $? -} - -download() { - # Initialize an empty variable for the Range header - RANGE_HEADER="" - - # If both RANGE_START and RANGE_END are set, add them to the Range header - if [ -n "$RANGE_START" ] && [ -n "$RANGE_END" ]; then - RANGE_HEADER="-H Range:bytes=$RANGE_START-$RANGE_END" - fi - - # Download the file - if [ "$APPEND" = "true" ]; then - curl $RANGE_HEADER https://aztec-ignition.s3-eu-west-2.amazonaws.com/MAIN%20IGNITION/monomial/transcript${1}.dat >> transcript${1}.dat - else - curl $RANGE_HEADER https://aztec-ignition.s3-eu-west-2.amazonaws.com/MAIN%20IGNITION/monomial/transcript${1}.dat > transcript${1}.dat - fi - -} - -for TRANSCRIPT in $(seq 0 $NUM); do - NUM=$(printf %02d $TRANSCRIPT) - if [ -f checksums ] && [ -z "$RANGE_START" ] && [ -z "$RANGE_END" ] ; then - checksum $NUM && continue - download $NUM - checksum $NUM || exit 1 - else - download $NUM - fi -done +./download_srs.sh "MAIN%20IGNITION" ignition/monomial $@ \ No newline at end of file diff --git a/barretenberg/cpp/srs_db/download_srs.sh b/barretenberg/cpp/srs_db/download_srs.sh new file mode 100755 index 00000000000..df7ffd03041 --- /dev/null +++ b/barretenberg/cpp/srs_db/download_srs.sh @@ -0,0 +1,52 @@ +#!/bin/sh +set -eu + +AWS_BUCKET=$1 +DESTINATION=$2 +NUM=${3:-19} +RANGE_START=${4:-} +RANGE_END=${5:-} +APPEND=${6:-"false"} + +mkdir -p "$DESTINATION" +cd "$DESTINATION" + +if command -v sha256sum > /dev/null; then + SHASUM=sha256sum +else + SHASUM="shasum -a 256" +fi + +checksum() { + grep transcript${1}.dat checksums | $SHASUM -c + return $? +} + +download() { + # Initialize an empty variable for the Range header + RANGE_HEADER="" + + # If both RANGE_START and RANGE_END are set, add them to the Range header + if [ -n "$RANGE_START" ] && [ -n "$RANGE_END" ]; then + RANGE_HEADER="-H Range:bytes=$RANGE_START-$RANGE_END" + fi + + # Download the file + if [ "$APPEND" = "true" ]; then + curl $RANGE_HEADER https://aztec-ignition.s3-eu-west-2.amazonaws.com/$AWS_BUCKET/monomial/transcript${1}.dat >> transcript${1}.dat + else + curl $RANGE_HEADER https://aztec-ignition.s3-eu-west-2.amazonaws.com/$AWS_BUCKET/monomial/transcript${1}.dat > transcript${1}.dat + fi + +} + +for TRANSCRIPT in $(seq 0 $NUM); do + NUM=$(printf %02d $TRANSCRIPT) + if [ -f checksums ] && [ -z "$RANGE_START" ] && [ -z "$RANGE_END" ] ; then + checksum $NUM && continue + download $NUM + checksum $NUM || exit 1 + else + download $NUM + fi +done diff --git a/barretenberg/cpp/srs_db/grumpkin/monomial/README.md b/barretenberg/cpp/srs_db/grumpkin/monomial/README.md deleted file mode 100644 index 9d3e53e8894..00000000000 --- a/barretenberg/cpp/srs_db/grumpkin/monomial/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# Quick-and-dirty Grumpkin transcript - -The Grumpkin transcript currently departs in structure from the BN254 -transcript in that: - - It does not contain a checksum - - It does not contain any g2 points (indeed, there is no grumpkin::g2). - - The transcript generation binary only produces a single transcript file. - If more than 504000 points are desired at some point, it is likely we will - need to a small refactor so that more files are created. - -A full-length transcript file containing 504000 points would have -BN254 transcript00.dat size: 322560412 -Grumpkin transcript00.dat size: 322560028 -322560028 - 322560412 = 384 = 256 + 128 - ^^^^^^^^^^^ ^^^^^^^^ - 2 g2 points checksum diff --git a/barretenberg/cpp/srs_db/ignition/monomial/checksums b/barretenberg/cpp/srs_db/ignition/monomial/checksums deleted file mode 100644 index 177b9441eb1..00000000000 --- a/barretenberg/cpp/srs_db/ignition/monomial/checksums +++ /dev/null @@ -1,21 +0,0 @@ -227512f4233e1b068e5dbfcf1a7fbb8b03d18050844eec29cb34af283fa6a7c9 g2.dat -0f94b39426f5a57d1a17742624f7d3170a32e19d6c5e16f202281439dbe48695 transcript00.dat -fe8919733b0ef279d31787f2c80e67a014dc50ad2790640157a7d411307784a3 transcript01.dat -0602f52fa0dd81a26cc5e1db5c068caf49118c30921238f41d8d16169041e15f transcript02.dat -b02f5c2b6d6017f90f3389ba2796465a11f582b49bbc900e76583a64bc8b32a5 transcript03.dat -66c14ce89a64345f70ac06f11bd327ce9b7aa3a33a7f27c8f094ba8a54374032 transcript04.dat -587bda8a743a4c21ff39bfbf269ed6f670ad2001e2490f001dd4ac6244d717ad transcript05.dat -08bddb87d7a0b063ff4e57e2a29d356398d2730fe33dddb54043c7e22015fe68 transcript06.dat -b3d36da80088d763490f752744036fc51b76ce623599926e72c96fac1a17ef0c transcript07.dat -c3f91827a2c0a187a77a01901371f8ca70731d3efbb1488135081318f86159bc transcript08.dat -98f98df7536245dcd1b40c12b0aea5f28a7fa6de9f4c7621164099fb57c9be64 transcript09.dat -8860f593a612f77f3ead23eefa201683b2576882365821e6fdcf40eab6ab20c6 transcript10.dat -b16a20941f3fe56f05f9df1fda79f5c43dc9f977ea0bcac9731431efafd3262c transcript11.dat -803c0bb9a43025602ec6ed4ddd303667da8c2e16a94410a49d5c95286737b00b transcript12.dat -c74beaa60e92d085a46b4f2336d452f4ba193a5304cef3a928eae7c87479b533 transcript13.dat -fc8a919e0629441dc1bce2e913aa87253a543cd343e67e00451bb98cecf9b701 transcript14.dat -1fa0c3f39221e83c205d081d7e09367e7a989323d0e6435232bc8f662491188f transcript15.dat -0e81089628ae03b869594dc4b170194f7f6fbfdecf6c9566f409b85697e8d5be transcript16.dat -82415a8e8a5c6106f199fddf0319151566a857fa2b11e9fe9c5d7568e5fc7612 transcript17.dat -099c74b4e257bd146323a161259b733d3c9c7702124cf52f620dc9f66ecaa11c transcript18.dat -dfa9e366e93c179325a969dc694986fa27392ba098e1941aeceae9f4683a3d66 transcript19.dat diff --git a/barretenberg/cpp/srs_db/ignition/monomial/g2.dat b/barretenberg/cpp/srs_db/ignition/monomial/g2.dat deleted file mode 100644 index 22bc78828b3..00000000000 --- a/barretenberg/cpp/srs_db/ignition/monomial/g2.dat +++ /dev/null @@ -1 +0,0 @@ -~#쓈YD;2Nո7N0Qz5NX&QDžJ^U?΢VG[Ba^"c*A ~/.ssh/build_instance_key chmod 600 ~/.ssh/build_instance_key + cp $BUILD_SYSTEM_PATH/remote/ssh_config ~/.ssh/config if [[ "$COMMIT_MESSAGE" == *"[ci debug]"* ]]; then echo export BUILD_SYSTEM_DEBUG=1 >> $BASH_ENV diff --git a/foundry/Dockerfile b/foundry/Dockerfile new file mode 100644 index 00000000000..65171755174 --- /dev/null +++ b/foundry/Dockerfile @@ -0,0 +1,35 @@ +FROM alpine:3.18 as build-environment + +ARG TARGETARCH +WORKDIR /opt + +RUN apk add clang lld curl build-base linux-headers git \ + && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > rustup.sh \ + && chmod +x ./rustup.sh \ + && ./rustup.sh -y + +RUN [[ "$TARGETARCH" = "arm64" ]] && echo "export CFLAGS=-mno-outline-atomics" >> $HOME/.profile || true + +WORKDIR /opt +RUN git clone --depth 1 --branch nightly-de33b6af53005037b463318d2628b5cfcaf39916 https://github.com/foundry-rs/foundry.git +WORKDIR /opt/foundry + +RUN source $HOME/.profile && cargo build --release \ + && mkdir out \ + && mv target/release/forge out/forge \ + && mv target/release/cast out/cast \ + && mv target/release/anvil out/anvil \ + && mv target/release/chisel out/chisel \ + && strip out/forge \ + && strip out/cast \ + && strip out/chisel \ + && strip out/anvil; + +FROM alpine:3.18 as foundry-client +RUN apk add --no-cache linux-headers git +COPY --from=build-environment /opt/foundry/out/forge /usr/local/bin/forge +COPY --from=build-environment /opt/foundry/out/cast /usr/local/bin/cast +COPY --from=build-environment /opt/foundry/out/anvil /usr/local/bin/anvil +COPY --from=build-environment /opt/foundry/out/chisel /usr/local/bin/chisel +RUN adduser -Du 1000 foundry +ENTRYPOINT ["/bin/sh", "-c"] \ No newline at end of file diff --git a/foundry/Earthfile b/foundry/Earthfile new file mode 100644 index 00000000000..24c8bf5512a --- /dev/null +++ b/foundry/Earthfile @@ -0,0 +1,11 @@ +VERSION 0.8 + +build: + ARG TARGETARCH + FROM DOCKERFILE . + SAVE IMAGE --push aztecprotocol/cache:foundry-nightly-de33b6af53005037b463318d2628b5cfcaf39916-$TARGETARCH + +get: + ARG TARGETARCH + # If this is failing, we need to run earthly --push +build + FROM aztecprotocol/cache:foundry-nightly-de33b6af53005037b463318d2628b5cfcaf39916-$TARGETARCH diff --git a/l1-contracts/Earthfile b/l1-contracts/Earthfile new file mode 100644 index 00000000000..7383b952ab2 --- /dev/null +++ b/l1-contracts/Earthfile @@ -0,0 +1,23 @@ +VERSION 0.8 +FROM ubuntu:lunar +RUN apt update && apt install curl git jq bash nodejs npm python3.11-full python3-pip -y + +# Use virtualenv, do not try to use pipx, it's not working. +RUN python3 -m venv /root/.venv +RUN /root/.venv/bin/pip3 install slither-analyzer==0.10.0 slitherin==0.5.0 +RUN curl -L https://foundry.paradigm.xyz | bash + +# Set env variables for foundry and venv +ENV PATH="${PATH}:/root/.foundry/bin:/root/.venv/bin" +RUN foundryup + +# Install yarn and solhint. +RUN npm install --global yarn solhint + +WORKDIR /build +COPY --keep-ts --dir lib scripts src terraform test *.json *.toml *.sh . + +build: + RUN git init && git add . && yarn lint && yarn slither && yarn slither-has-diff + RUN forge clean && forge fmt --check && forge build && forge test + SAVE ARTIFACT out diff --git a/noir-projects/.earthlyignore b/noir-projects/.earthlyignore new file mode 100644 index 00000000000..f2a4093411b --- /dev/null +++ b/noir-projects/.earthlyignore @@ -0,0 +1 @@ +**/target \ No newline at end of file diff --git a/noir-projects/Earthfile b/noir-projects/Earthfile new file mode 100644 index 00000000000..85673639c6b --- /dev/null +++ b/noir-projects/Earthfile @@ -0,0 +1,24 @@ +VERSION 0.8 +FROM ubuntu:lunar + +# Install nargo +COPY ../noir/+nargo/nargo /usr/bin/nargo +# Install transpiler +COPY ../avm-transpiler/+build/avm-transpiler /usr/bin/avm-transpiler + +WORKDIR /build +# Copy source. +COPY --dir aztec-nr noir-contracts noir-protocol-circuits . + +build: + RUN cd noir-contracts && NARGO=nargo TRANSPILER=avm-transpiler ./bootstrap.sh + RUN cd noir-protocol-circuits && NARGO=nargo ./bootstrap.sh + SAVE ARTIFACT aztec-nr + SAVE ARTIFACT noir-contracts + SAVE ARTIFACT noir-protocol-circuits + +test: + FROM +build + RUN cd noir-protocol-circuits && nargo test --silence-warnings + RUN cd aztec-nr && nargo test --silence-warnings + RUN cd noir-contracts && nargo test --silence-warnings diff --git a/noir-projects/noir-contracts/bootstrap.sh b/noir-projects/noir-contracts/bootstrap.sh index 06c1df51bfb..eccec0ff99e 100755 --- a/noir-projects/noir-contracts/bootstrap.sh +++ b/noir-projects/noir-contracts/bootstrap.sh @@ -16,10 +16,12 @@ if [ -n "$CMD" ]; then fi echo "Compiling contracts..." -../../noir/noir-repo/target/release/nargo compile --silence-warnings +NARGO=${NARGO:-../../noir/noir-repo/target/release/nargo} +$NARGO compile --silence-warnings echo "Transpiling avm contracts..." for contract_json in target/avm_test_*.json; do echo Transpiling $contract_json... - ../../avm-transpiler/target/release/avm-transpiler $contract_json $contract_json + TRANSPILER=${TRANSPILER:-../../avm-transpiler/target/release/avm-transpiler} + $TRANSPILER $contract_json $contract_json done \ No newline at end of file diff --git a/noir-projects/noir-protocol-circuits/bootstrap.sh b/noir-projects/noir-protocol-circuits/bootstrap.sh index b7af2ad14f6..e647f07b80f 100755 --- a/noir-projects/noir-protocol-circuits/bootstrap.sh +++ b/noir-projects/noir-protocol-circuits/bootstrap.sh @@ -16,4 +16,5 @@ if [ -n "$CMD" ]; then fi echo "Compiling protocol circuits..." -../../noir/noir-repo/target/release/nargo compile --silence-warnings \ No newline at end of file +NARGO=${NARGO:-../../noir/noir-repo/target/release/nargo} +$NARGO compile --silence-warnings \ No newline at end of file diff --git a/noir/Earthfile b/noir/Earthfile new file mode 100644 index 00000000000..f2d91a4cd7b --- /dev/null +++ b/noir/Earthfile @@ -0,0 +1,111 @@ +VERSION 0.8 + + +nargo: + FROM rust:bullseye + RUN apt update && apt install -y libc++1 + WORKDIR /build + # Relevant source (TODO finer-grained 'tooling') + COPY --keep-ts --dir \ + noir-repo/acvm-repo \ + noir-repo/aztec_macros \ + noir-repo/compiler \ + noir-repo/noir_stdlib \ + noir-repo/tooling \ + noir-repo/test_programs \ + noir-repo/Cargo.lock \ + noir-repo/Cargo.toml \ + noir-repo + + # TODO(AD) is this OK as a content hash? + ENV COMMIT_HASH=$(find . -type f -exec sha256sum {} ';' | sort | sha256sum | awk '{print $1}') + COPY --keep-ts ./scripts/bootstrap_native.sh ./scripts/bootstrap_native.sh + RUN ./scripts/bootstrap_native.sh + SAVE ARTIFACT /build/noir-repo/target/release/nargo nargo + SAVE ARTIFACT /build/noir-repo/target/release/acvm acvm + +packages: + FROM node:20 + RUN curl https://sh.rustup.rs -sSf | bash -s -- -y + RUN echo 'source $HOME/.cargo/env' >> $HOME/.bashrc + RUN apt update && apt install -y jq libc++1 + WORKDIR /build + + # Relevant source (TODO finer-grained) + COPY --keep-ts --dir \ + noir-repo/acvm-repo \ + noir-repo/aztec_macros \ + noir-repo/compiler \ + noir-repo/docs \ + noir-repo/noir_stdlib \ + noir-repo/scripts \ + noir-repo/test_programs \ + noir-repo/tooling \ + noir-repo/Cargo.lock \ + noir-repo/.yarnrc.yml \ + noir-repo/.yarn \ + noir-repo/yarn.lock \ + noir-repo/package.json \ + noir-repo/LICENSE* \ + noir-repo/*.toml \ + noir-repo/*.json \ + noir-repo/*.js \ + noir-repo/.github \ + noir-repo/.envrc \ + noir-repo + + COPY --keep-ts noir-repo/.github/scripts noir-repo/.github/scripts + COPY --keep-ts ./scripts/bootstrap_packages.sh ./scripts/bootstrap_packages.sh + # TODO(AD) is this OK as a content hash? + ENV COMMIT_HASH=$(find . -type f -exec sha256sum {} ';' | sort | sha256sum | awk '{print $1}') + RUN PATH="/root/.cargo/bin:$PATH" ./scripts/bootstrap_packages.sh + SAVE ARTIFACT packages + +run: + # When running the container, mount the users home directory to same location. + FROM ubuntu:lunar + # Install Tini as nargo doesn't handle signals properly. + # Install git as nargo needs it to clone. + RUN apt-get update && apt-get install -y git tini && rm -rf /var/lib/apt/lists/* && apt-get clean + COPY +build/. /build + ENTRYPOINT ["/usr/bin/tini", "--", "/build/nargo"] + +build: + BUILD +nargo + BUILD +packages + +# TOOD +# test-packages +# FROM aztecprotocol/noir AS noir + +# FROM node:20 AS builder +# COPY --from=noir /usr/src/noir/noir-repo/target/release /usr/src/noir/noir-repo/target/release +# ENV PATH=${PATH}:/usr/src/noir/noir-repo/target/release +# RUN curl https://sh.rustup.rs -sSf | bash -s -- -y +# RUN echo 'source $HOME/.cargo/env' >> $HOME/.bashrc +# ENV PATH=/root/.cargo/bin:${PATH} +# RUN apt update && apt install -y jq libc++1 +# ARG COMMIT_HASH +# ENV COMMIT_HASH=${COMMIT_HASH} + +# WORKDIR /usr/src/noir +# COPY . . +# RUN ./scripts/test_js_packages.sh + +# # Don't waste time pushing a huge container back to ECR as nothing needs the output. +# FROM scratch +# COPY --from=builder /usr/src/noir/README.md /usr/src/noir/README.md + +# TODO +# test: +# FROM rust:bullseye +# ARG COMMIT_HASH +# ENV COMMIT_HASH=${COMMIT_HASH} +# RUN apt update && apt install -y libc++1 +# WORKDIR /usr/src/noir +# COPY . . +# RUN ./scripts/test_native.sh + +# # Don't waste time pushing a huge container back to ECR as nothing needs the output. +# FROM scratch +# COPY --from=0 /usr/src/noir/README.md /usr/src/noir/README.md diff --git a/noir/noir-repo/.github/scripts/wasm-bindgen-install.sh b/noir/noir-repo/.github/scripts/wasm-bindgen-install.sh index a548372ee2c..20908003693 100755 --- a/noir/noir-repo/.github/scripts/wasm-bindgen-install.sh +++ b/noir/noir-repo/.github/scripts/wasm-bindgen-install.sh @@ -6,7 +6,7 @@ cd $(dirname "$0") ./cargo-binstall-install.sh # Install wasm-bindgen-cli. -if [ "$(wasm-bindgen --version | cut -d' ' -f2)" != "0.2.86" ]; then +if [ "$(wasm-bindgen --version &> /dev/null | cut -d' ' -f2)" != "0.2.86" ]; then echo "Building wasm-bindgen..." cargo binstall wasm-bindgen-cli@0.2.86 --force --no-confirm fi diff --git a/scripts/earthly b/scripts/earthly new file mode 100755 index 00000000000..ff0137b8415 --- /dev/null +++ b/scripts/earthly @@ -0,0 +1,31 @@ + +#!/usr/bin/env bash +[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace + +# Aztec build/test/bench tool +# Thin wrapper for earthly that helps with building targets. +# Adds autodownloading earthly and timing code. +# Usage: +# Go to folder, e.g. docs, use az to build +# To run a target in the Earthfile, use 'az +'. +# Spports all commands 'earthly' can take. + +set -euo pipefail + +mkdir -p $(dirname $0)/.earthly + +EARTHLY=$(dirname $0)/.earthly/earthly +EARTHLY_CONFIG=$(dirname $0)/earthly-config.yml + +if ! [ -f "$EARTHLY" ] ; then + wget https://github.com/earthly/earthly/releases/latest/download/earthly-linux-amd64 -O "$EARTHLY"; + chmod +x "$EARTHLY" +fi + +EARTHLY_FLAGS="-P --disable-remote-registry-proxy --use-inline-cache --save-inline-cache" +if [ $# -eq 0 ]; then + # By default, try for a +build target + "$EARTHLY" $EARTHLY_FLAGS +build +else + "$EARTHLY" $EARTHLY_FLAGS $@ +fi \ No newline at end of file diff --git a/scripts/earthly-config.yml b/scripts/earthly-config.yml new file mode 100644 index 00000000000..c2b6747834f --- /dev/null +++ b/scripts/earthly-config.yml @@ -0,0 +1,3 @@ +global: + cache_size_mb: 100000 + container_frontend: docker-shell diff --git a/scripts/earthly-timed b/scripts/earthly-timed new file mode 100755 index 00000000000..62ad856b734 --- /dev/null +++ b/scripts/earthly-timed @@ -0,0 +1,55 @@ + +#!/usr/bin/env bash +[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace + +# Aztec build/test/bench tool +# Thin wrapper for earthly that helps with building targets. +# Adds autodownloading earthly and timing code. +# Usage: +# Go to folder, e.g. docs, use az to build +# To run a target in the Earthfile, use 'az +'. +# Spports all commands 'earthly' can take. + +set -euo pipefail + +export FORCE_COLOR=1 +add_timestamps() { + gray=$(tput setaf 8) + normal=$(tput sgr0) + while IFS= read -r line; do printf "${gray}%(%H:%M:%S)T${normal} " ; echo "$line" ; done +} +report_time() { + end_time=$(date +%s) + duration=$((end_time - start_time)) + gray=$(tput setaf 8) + normal=$(tput sgr0) + local hours=$((duration / 3600)) + local minutes=$(( (duration % 3600) / 60 )) + local seconds=$((duration % 60)) + + printf "${gray}" + if (( hours > 0 )); then + printf "%d hours, %d minutes, and %d seconds" $hours $minutes $seconds + elif (( minutes > 0 )); then + printf "%d minutes and %d seconds" $minutes $seconds + else + printf "%d seconds" $seconds + fi + printf "${normal}\n" +} +# on finish +trap report_time EXIT +start_time=$(date +%s) + +INTERACTIVE=${INTERACTIVE:-false} +for arg in "$@"; do + if [ "$arg" = "-i" ] || [ "$arg" = "--interactive" ]; then + INTERACTIVE=true + break + fi +done +if [ $INTERACTIVE = true ] ; then + $(dirname $0)/earthly $@ +else + $(dirname $0)/earthly $@ 2>&1 | add_timestamps >&2 +fi \ No newline at end of file diff --git a/yarn-project/.earthlyignore b/yarn-project/.earthlyignore new file mode 100644 index 00000000000..25434442e60 --- /dev/null +++ b/yarn-project/.earthlyignore @@ -0,0 +1,56 @@ +# Must include the .gitignore for all child projects as this is used by Earthly +# Note due to how we use Eartlhy each .gitignore MUST accompany any earthfile that might actually copy these artifacts +**/Earthfile +**/Readme.md +**/Dockerfile* +**/docker-compose*.yml + +# .gitignore contents: +**/dest +**/node_modules +**/.cache +.env* +*.log +*.swp +**/.tsbuildinfo +**/tsconfig.tsbuildinfo +**/.eslintcache +**/target +accounts/src/artifacts +aztec-faucet/data* +aztec-node/data* +aztec-js/src/account_contract/artifacts +aztec/log +circuits.js/fixtures/*.json +docs/dist +end-to-end/addresses.json +end-to-end/log +end-to-end/data +end-to-end/src/web/main.js +end-to-end/src/web/main.js.LICENSE.txt +entry-points/src/artifacts +l1-contracts/generated +noir-compiler/target/ +noir-compiler/proofs/ +noir-compiler/Prover.toml +noir-compiler/Verifier.toml +noir-compiler/noir-protocol-circuits-types/.gitignore wow +noir-compiler/proofs/ +noir-compiler/Prover.toml +noir-compiler/Verifier.toml +noir-compiler/src/target +noir-compiler/src/crs +noir-compiler/src/types +protocol-contracts/src/artifacts +scripts/tmp +noir-contracts.js/src +noir-contracts.js/artifacts/ +noir-contracts.js/codegenCache.json + + +.yarn/* +!.yarn/patches +!.yarn/plugins +!.yarn/releases +!.yarn/sdks +!.yarn/versions diff --git a/yarn-project/.gitignore b/yarn-project/.gitignore index a8ce42c3200..1db6722f9c9 100644 --- a/yarn-project/.gitignore +++ b/yarn-project/.gitignore @@ -1,15 +1,46 @@ -dest -node_modules -.cache +# To be maintained along with .earthlyinore +**/dest +**/node_modules +**/.cache .env* *.log *.swp -.tsbuildinfo -tsconfig.tsbuildinfo -.eslintcache -simulator/target -.debounce-* -.tsc.pid +**/.tsbuildinfo +**/tsconfig.tsbuildinfo +**/.eslintcache +**/target +**/.debounce-* +**/.tsc.pid +accounts/src/artifacts +aztec-faucet/data* +aztec-node/data* +aztec-js/src/account_contract/artifacts +aztec/log +circuits.js/fixtures/*.json +docs/dist +end-to-end/addresses.json +end-to-end/log +end-to-end/data +end-to-end/src/web/main.js +end-to-end/src/web/main.js.LICENSE.txt +entry-points/src/artifacts +l1-contracts/generated +noir-compiler/target/ +noir-compiler/proofs/ +noir-compiler/Prover.toml +noir-compiler/Verifier.toml +noir-compiler/noir-protocol-circuits-types/.gitignore wow +noir-compiler/proofs/ +noir-compiler/Prover.toml +noir-compiler/Verifier.toml +noir-compiler/src/target +noir-compiler/src/crs +noir-compiler/src/types +protocol-contracts/src/artifacts +scripts/tmp +noir-contracts.js/src +noir-contracts.js/artifacts/ +noir-contracts.js/codegenCache.json .yarn/* !.yarn/patches diff --git a/yarn-project/Earthfile b/yarn-project/Earthfile new file mode 100644 index 00000000000..d2dba999259 --- /dev/null +++ b/yarn-project/Earthfile @@ -0,0 +1,114 @@ +VERSION 0.8 +FROM node:18.19.0 +RUN apt update && apt install -y jq curl perl && rm -rf /var/lib/apt/lists/* && apt-get clean + +# copy bb-js and noir-packages +COPY ../barretenberg/ts/+build/build /build/barretenberg/ts +COPY ../noir/+packages/packages /build/noir/packages +# install acvm binary +COPY ../noir/+nargo/acvm /usr/bin/acvm +COPY --dir ../noir-projects/+build/. /build/noir-projects +COPY ../l1-contracts/+build/out /build/l1-contracts/out + +WORKDIR /build/yarn-project +# copy source +COPY --keep-ts --dir * *.json .yarn .yarnrc.yml . + +# We install a symlink to yarn-project's node_modules at a location that all portalled packages can find as they +# walk up the tree as part of module resolution. The supposedly idiomatic way of supporting module resolution +# correctly for portalled packages, is to use --preserve-symlinks when running node. +# This does kind of work, but jest doesn't honor it correctly, so this seems like a neat workaround. +# Also, --preserve-symlinks causes duplication of portalled instances such as bb.js, and breaks the singleton logic +# by initialising the module more than once. So at present I don't see a viable alternative. +RUN ln -s /build/yarn-project/node_modules /build/node_modules + +# Target for main build process +build: + ARG EARTHLY_CI + # TODO: Replace puppeteer with puppeteer-core to avoid this. + # TODO encapsulate in bash script for cleanliness + ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true + IF $EARTHLY_CI # Don't cache CI builds. + RUN ./bootstrap.sh && yarn workspaces focus @aztec/cli @aztec/aztec --production && yarn cache clean + ELSE + ENV YARN_CACHE_FOLDER /build/yarn-cache + # Use a mount for incremental builds locally. + RUN --mount type=cache,target=/build/node_modules_cache --mount type=cache,target=/build/yarn-cache \ + mkdir -p node_modules_cache && \ + mv node_modules_cache node_modules && \ + ./bootstrap.sh && \ + yarn workspaces focus @aztec/cli @aztec/aztec --production && \ + cp -r node_modules node_modules_cache + END + SAVE ARTIFACT /build + +# TODO versioning flow at end before publish? +# ENV COMMIT_TAG=$EARTHLY_BUILD_SHA +# RUN ./scripts/version_packages.sh + +# run: +# FROM node:18.19.1-slim +# ARG COMMIT_TAG="" +# ENV COMMIT_TAG=$COMMIT_TAG +# COPY --from=builder /build /build +# WORKDIR /build/yarn-project +# ENTRYPOINT ["yarn"] + +aztec: + FROM +build + # ENV vars for using native ACVM simulation + ENV ACVM_BINARY_PATH="/usr/bin/acvm" ACVM_WORKING_DIRECTORY="/tmp/acvm" + ENTRYPOINT ["node", "--no-warnings", "/build/yarn-project/aztec/dest/bin/index.js"] + EXPOSE 8080 + # TODO(AD) the following are the biggest node modules bundled, should they be deleted as they are build tools? + # 25840 @jest + # 31544 typescript + # 62368 @types + + # The version has been updated in yarn-project. + # Adding COMMIT_TAG here to rebuild versioned image. + ARG COMMIT_TAG="" + +end-to-end: + # compilation artifacts for end-to-end + # TODO encapsulate in bash script for cleanliness + ARG EARTHLY_CI + IF $EARTHLY_CI # Don't cache CI builds. + RUN ./bootstrap.sh && \ + yarn workspace @aztec/end-to-end run build:web && \ + yarn workspaces focus @aztec/end-to-end --production \ + && yarn cache clean + ELSE + ENV YARN_CACHE_FOLDER /yarn_cache + # TODO copy to other targets + # Use a mount for incremental builds locally. + # We cache NPM installs, typescript metadata and dest folders. + # TODO(AD): find safe way to cache noir contracts + RUN --mount type=cache,target=/node_cache \ + --mount type=cache,target=/yarn_cache \ + mkdir -p /node_cache/node_modules && mv /node_cache/node_modules . && \ + cp -r /node_cache/* . >/dev/null || echo "fresh build" && \ + ./bootstrap.sh && \ + yarn workspace @aztec/end-to-end run build:web && \ + yarn workspaces focus @aztec/end-to-end --production && \ + cp -r node_modules /node_cache/node_modules && \ + cp -r --parents */dest /node_cache && \ + find . -name '*.tsbuildinfo' -exec cp --parents {} /node_cache \; + END + SAVE ARTIFACT /build + + # Build web bundle for browser tests + RUN yarn workspace @aztec/end-to-end run build:web + RUN yarn workspaces focus @aztec/end-to-end --production && yarn cache clean + SAVE ARTIFACT /build + +end-to-end-minimal: + # end to end test runner + BUILD +aztec + FROM node:18.19.1-slim + RUN apt-get update && apt-get install jq chromium -y + ENV CHROME_BIN="/usr/bin/chromium" + COPY +end-to-end/build /build + WORKDIR /build/yarn-project/end-to-end + ENTRYPOINT ["yarn", "test"] + SAVE IMAGE aztecprotocol/cache:end-to-end diff --git a/yarn-project/accounts/.gitignore b/yarn-project/accounts/.gitignore deleted file mode 100644 index 7912fc48c7d..00000000000 --- a/yarn-project/accounts/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/src/artifacts diff --git a/yarn-project/aztec-faucet/.gitignore b/yarn-project/aztec-faucet/.gitignore deleted file mode 100644 index 81efe293f26..00000000000 --- a/yarn-project/aztec-faucet/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/data* diff --git a/yarn-project/aztec-node/.gitignore b/yarn-project/aztec-node/.gitignore deleted file mode 100644 index 81efe293f26..00000000000 --- a/yarn-project/aztec-node/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/data* diff --git a/yarn-project/aztec.js/.gitignore b/yarn-project/aztec.js/.gitignore deleted file mode 100644 index 03b565b6c24..00000000000 --- a/yarn-project/aztec.js/.gitignore +++ /dev/null @@ -1 +0,0 @@ -src/account_contract/artifacts \ No newline at end of file diff --git a/yarn-project/aztec/.gitignore b/yarn-project/aztec/.gitignore deleted file mode 100644 index 31dbbff57c2..00000000000 --- a/yarn-project/aztec/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/log diff --git a/yarn-project/circuits.js/.gitignore b/yarn-project/circuits.js/.gitignore deleted file mode 100644 index 6fc41e1039e..00000000000 --- a/yarn-project/circuits.js/.gitignore +++ /dev/null @@ -1 +0,0 @@ -fixtures/*.json diff --git a/yarn-project/docs/.gitignore b/yarn-project/docs/.gitignore deleted file mode 100644 index 53c37a16608..00000000000 --- a/yarn-project/docs/.gitignore +++ /dev/null @@ -1 +0,0 @@ -dist \ No newline at end of file diff --git a/yarn-project/end-to-end/.gitignore b/yarn-project/end-to-end/.gitignore deleted file mode 100644 index 48645e85aab..00000000000 --- a/yarn-project/end-to-end/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -addresses.json -/log -/data -/src/web/main.js -/src/web/main.js.LICENSE.txt \ No newline at end of file diff --git a/yarn-project/end-to-end/Earthfile b/yarn-project/end-to-end/Earthfile new file mode 100644 index 00000000000..c7228768a72 --- /dev/null +++ b/yarn-project/end-to-end/Earthfile @@ -0,0 +1,151 @@ + +VERSION 0.8 + +get-aztec: + FROM ubuntu:lunar + # for use_aztec_image to work, need to run earthly +build first or have image copied + ARG use_aztec_image=false + IF [ "$use_aztec_image" != false ] + FROM aztecprotocol/cache:aztec + ELSE + FROM ../+aztec + END + +get-end-to-end: + FROM ubuntu:lunar + # for use_end_to_end_image to work, need to run earthly +build first or have image copied + ARG use_end_to_end_image=false + IF [ $use_end_to_end_image != false ] + FROM aztecprotocol/cache:end-to-end + ELSE + FROM ../+end-to-end-minimal + END + +E2E_TEST: + FUNCTION + ARG test + ARG compose_file=./scripts/docker-compose.yml + LOCALLY + ENV TEST=$test + WITH DOCKER \ + --load aztecprotocol/aztec:latest=+get-aztec \ + --load aztecprotocol/end-to-end:latest=+get-end-to-end \ + --load ghcr.io/foundry-rs/foundry:nightly-de33b6af53005037b463318d2628b5cfcaf39916=../../foundry/+get + # Run our docker compose, ending whenever sandbox ends, filtering out noisy eth_getLogs + RUN docker compose -f $compose_file up --exit-code-from=sandbox --force-recreate + END + +# we could use a parameterized target, but these just print cleaner in earthly log + +e2e-block-building: + DO +E2E_TEST --test=e2e_block_building.test.ts + +e2e-nested-contract: + DO +E2E_TEST --test=e2e_nested_contract.test.ts + +e2e-static-calls: + DO +E2E_TEST --test=e2e_static_calls.test.ts + +e2e-delegate-calls: + DO +E2E_TEST --test=e2e_delegate_calls.test.ts + +e2e-non-contract-account: + DO +E2E_TEST --test=e2e_non_contract_account.test.ts + +e2e-cross-chain-messaging: + DO +E2E_TEST --test=e2e_cross_chain_messaging.test.ts + +e2e-crowdfunding-and-claim: + DO +E2E_TEST --test=e2e_crowdfunding_and_claim.test.ts + +e2e-public-cross-chain-messaging: + DO +E2E_TEST --test=e2e_public_cross_chain_messaging.test.ts + +e2e-public-to-private-messaging: + DO +E2E_TEST --test=e2e_public_to_private_messaging.test.ts + +e2e-account-contracts: + DO +E2E_TEST --test=e2e_account_contracts.test.ts + +e2e-escrow-contract: + DO +E2E_TEST --test=e2e_escrow_contract.test.ts + +e2e-inclusion-proofs-contract: + DO +E2E_TEST --test=e2e_inclusion_proofs_contract.test.ts + +e2e-pending-note-hashes-contract: + DO +E2E_TEST --test=e2e_pending_note_hashes_contract.test.ts + +e2e-ordering: + DO +E2E_TEST --test=e2e_ordering.test.ts + +uniswap-trade-on-l1-from-l2: + DO +E2E_TEST --test=uniswap_trade_on_l1_from_l2.test.ts + +integration-archiver-l1-to-l2: + DO +E2E_TEST --test=integration_archiver_l1_to_l2.test.ts + +integration-l1-publisher: + DO +E2E_TEST --test=integration_l1_publisher.test.ts + +e2e-cli: + DO +E2E_TEST --test=e2e_cli.test.ts + +e2e-persistence: + DO +E2E_TEST --test=e2e_persistence.test.ts --compose_file=./scripts/docker-compose-no-sandbox.yml + +e2e-browser: + DO +E2E_TEST --test=e2e_aztec_js_browser.test.ts + +e2e-card-game: + DO +E2E_TEST --test=e2e_card_game.test.ts + +e2e-avm-simulator: + ENV AVM_ENABLED=1 + DO +E2E_TEST --test=e2e_avm_simulator.test.ts + +e2e-fees: + DO +E2E_TEST --test=e2e_fees.test.ts + +e2e-dapp-subscription: + DO +E2E_TEST --test=e2e_dapp_subscription.test.ts + +pxe: + DO +E2E_TEST --test=pxe_sandbox.test.ts + +cli-docs-sandbox: + DO +E2E_TEST --test=cli_docs_sandbox.test.ts + +e2e-docs-examples: + ENV AVM_ENABLED=1 + DO +E2E_TEST --test=docs_examples_test.ts + +test-all: + BUILD +e2e-state-vars + BUILD +e2e-block-building + BUILD +e2e-nested-contract + BUILD +e2e-static-calls + BUILD +e2e-delegate-calls + BUILD +e2e-non-contract-account + BUILD +e2e-cross-chain-messaging + BUILD +e2e-crowdfunding-and-claim + BUILD +e2e-public-cross-chain-messaging + BUILD +e2e-public-to-private-messaging + BUILD +e2e-account-contracts + BUILD +e2e-escrow-contract + BUILD +e2e-inclusion-proofs-contract + BUILD +e2e-pending-note-hashes-contract + BUILD +e2e-ordering + BUILD +uniswap-trade-on-l1-from-l2 + BUILD +integration-archiver-l1-to-l2 + BUILD +integration-l1-publisher + BUILD +e2e-cli + BUILD +e2e-persistence + BUILD +e2e-browser + BUILD +e2e-card-game + BUILD +e2e-avm-simulator + BUILD +e2e-fees + BUILD +e2e-dapp-subscription + BUILD +pxe + BUILD +cli-docs-sandbox + BUILD +e2e-docs-examples diff --git a/yarn-project/end-to-end/scripts/docker-compose.yml b/yarn-project/end-to-end/scripts/docker-compose.yml index 0a73d344095..114fd5f9ad8 100644 --- a/yarn-project/end-to-end/scripts/docker-compose.yml +++ b/yarn-project/end-to-end/scripts/docker-compose.yml @@ -5,10 +5,12 @@ services: entrypoint: > sh -c ' if [ -n "$FORK_BLOCK_NUMBER" ] && [ -n "$FORK_URL" ]; then - exec anvil -p 8545 --host 0.0.0.0 --chain-id 31337 --fork-url "$FORK_URL" --fork-block-number "$FORK_BLOCK_NUMBER" + { anvil -p 8545 --host 0.0.0.0 --chain-id 31337 --fork-url "$FORK_URL" --fork-block-number "$FORK_BLOCK_NUMBER" ; echo $$? > .status ; } | grep -v eth_getLogs else - exec anvil -p 8545 --host 0.0.0.0 --chain-id 31337 - fi' + { anvil -p 8545 --host 0.0.0.0 --chain-id 31337 ; echo $$? > .status ; } | grep -v eth_getLogs + fi + # final status + grep -q '^0$$' .status' ports: - '8545:8545' @@ -40,6 +42,7 @@ services: PXE_URL: http://sandbox:8080 command: ${TEST:-./src/e2e_deploy_contract.test.ts} volumes: + # TODO(AD) currently earthly uses /build instead of /usr/src - ../log:/usr/src/yarn-project/end-to-end/log:rw depends_on: - sandbox diff --git a/yarn-project/entrypoints/.gitignore b/yarn-project/entrypoints/.gitignore deleted file mode 100644 index 7912fc48c7d..00000000000 --- a/yarn-project/entrypoints/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/src/artifacts diff --git a/yarn-project/l1-artifacts/.gitignore b/yarn-project/l1-artifacts/.gitignore deleted file mode 100644 index dc9b2375c7a..00000000000 --- a/yarn-project/l1-artifacts/.gitignore +++ /dev/null @@ -1 +0,0 @@ -generated \ No newline at end of file diff --git a/yarn-project/noir-compiler/.gitignore b/yarn-project/noir-compiler/.gitignore deleted file mode 100644 index 4e3795cbce5..00000000000 --- a/yarn-project/noir-compiler/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -target/ -proofs/ -Prover.toml -Verifier.toml \ No newline at end of file diff --git a/yarn-project/noir-contracts.js/.gitignore b/yarn-project/noir-contracts.js/.gitignore deleted file mode 100644 index afe0a2a36cb..00000000000 --- a/yarn-project/noir-contracts.js/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -target/ -/src -artifacts/ -codegenCache.json diff --git a/yarn-project/noir-protocol-circuits-types/.gitignore b/yarn-project/noir-protocol-circuits-types/.gitignore deleted file mode 100644 index bc5c9c9dc28..00000000000 --- a/yarn-project/noir-protocol-circuits-types/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -proofs/ -Prover.toml -Verifier.toml -src/target -src/crs -src/types diff --git a/yarn-project/package.json b/yarn-project/package.json index 5d83d4bbb73..5e5fbfb0a68 100644 --- a/yarn-project/package.json +++ b/yarn-project/package.json @@ -15,7 +15,7 @@ "build:fast": "yarn generate && tsc -b", "build:dev": "./watch.sh", "generate": "FORCE_COLOR=true yarn workspaces foreach --parallel --topological-dev --verbose run generate", - "clean": "yarn workspaces foreach -p -v run clean" + "clean": "yarn workspaces foreach -p -v run clean " }, "workspaces": [ "accounts", diff --git a/yarn-project/protocol-contracts/.gitignore b/yarn-project/protocol-contracts/.gitignore deleted file mode 100644 index 7912fc48c7d..00000000000 --- a/yarn-project/protocol-contracts/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/src/artifacts diff --git a/yarn-project/scripts/.gitignore b/yarn-project/scripts/.gitignore deleted file mode 100644 index ceeb05b4108..00000000000 --- a/yarn-project/scripts/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/tmp diff --git a/yarn-project/scripts/version_packages.sh b/yarn-project/scripts/version_packages.sh index bf631bd8ac1..d3865f324e7 100755 --- a/yarn-project/scripts/version_packages.sh +++ b/yarn-project/scripts/version_packages.sh @@ -4,5 +4,7 @@ set -eu if [ -n "$COMMIT_TAG" ]; then for workspace in $(yarn workspaces list --json | jq -r '.location'); do (cd $workspace && jq --arg v $COMMIT_TAG '.version = $v' package.json > _temp && mv _temp package.json) + # allow for versioning already-built packages + (cd $workspace && [ -f dest/package.json ] && jq --arg v $COMMIT_TAG '.version = $v' dest/package.json > _temp && mv _temp dest/package.json) done fi \ No newline at end of file