diff --git a/.circleci/config.yml b/.circleci/config.yml index 913e0cd6bce..1a69da686a0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1313,7 +1313,10 @@ workflows: - barretenberg-stdlib-tests: *bb_test - barretenberg-stdlib-recursion-ultra-tests: *bb_test - barretenberg-acir-tests-bb: *bb_acir_tests - - barretenberg-acir-tests-bb-sol: *bb_acir_tests + - barretenberg-acir-tests-bb-sol: + requires: + - barretenberg-x86_64-linux-clang-sol + <<: *bb_acir_tests - barretenberg-docs: *defaults - bb-js: requires: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000000..de83032b8fd --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,46 @@ +name: Run CI with Earthly +on: + push: + branches: + - master + pull_request: {} + workflow_dispatch: {} + +jobs: + ci: + runs-on: ubuntu-latest + # run ci for both x86_64 and arm64 + strategy: {matrix: {environment: [x86, arm]}} + # cancel if reran on same PR if exists, otherwise if on same commit + concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ matrix.environment }} + cancel-in-progress: true + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + submodules: recursive + + - name: Setup + run: | + mkdir -p ~/.ssh + echo DOCKER_HOST=ssh://build-instance-${{ matrix.environment }}.aztecprotocol.com >> $GITHUB_ENV + echo ${{ secrets.DOCKERHUB_PASSWORD}} | docker login -u aztecprotocolci --password-stdin + echo ${{ secrets.BUILD_INSTANCE_SSH_KEY }} | base64 -d > ~/.ssh/build_instance_key + chmod 600 ~/.ssh/build_instance_key + cat > ~/.ssh/config < $ $ diff --git a/barretenberg/cpp/src/barretenberg/barretenberg.hpp b/barretenberg/cpp/src/barretenberg/barretenberg.hpp deleted file mode 100644 index 659f3f2d115..00000000000 --- a/barretenberg/cpp/src/barretenberg/barretenberg.hpp +++ /dev/null @@ -1,74 +0,0 @@ -#pragma once - -// External Barretenberg C++ API -#include "common/bbmalloc.hpp" -#include "common/container.hpp" -#include "common/map.hpp" -#include "common/mem.hpp" -#include "common/serialize.hpp" -#include "common/streams.hpp" -#include "common/throw_or_abort.hpp" -#include "crypto/blake2s/blake2s.hpp" -#include "crypto/blake3s/blake3s.hpp" -#include "crypto/ecdsa/ecdsa.hpp" -#include "crypto/generators/generator_data.hpp" -#include "crypto/keccak/keccak.hpp" -#include "crypto/pedersen_commitment/pedersen.hpp" -#include "crypto/pedersen_hash/pedersen.hpp" -#include "crypto/poseidon2/poseidon2.hpp" -#include "crypto/schnorr/schnorr.hpp" -#include "crypto/sha256/sha256.hpp" -#include "ecc/curves/bn254/fq.hpp" -#include "ecc/curves/bn254/fr.hpp" -#include "ecc/curves/bn254/g1.hpp" -#include "ecc/curves/grumpkin/grumpkin.hpp" -#include "numeric/random/engine.hpp" -#include "numeric/uint256/uint256.hpp" -#include "plonk/proof_system/types/proof.hpp" -#include "plonk/proof_system/verification_key/verification_key.hpp" -#include "proof_system/circuit_builder/ultra_circuit_builder.hpp" -#include "proof_system/types/circuit_type.hpp" -// TODO(https://github.com/AztecProtocol/barretenberg/issues/491): -// consider helper header(s) for serialization and other non-stdlib includes -// - possibly: common, serialize, srs, transcript -#include "serialize/cbind.hpp" -#include "serialize/cbind_fwd.hpp" -#include "serialize/msgpack.hpp" -#include "serialize/test_helper.hpp" -#include "srs/global_crs.hpp" -#include "stdlib/commitment/pedersen/pedersen.hpp" -#include "stdlib/encryption/ecdsa/ecdsa.hpp" -#include "stdlib/encryption/schnorr/schnorr.hpp" -#include "stdlib/hash/blake2s/blake2s.hpp" -#include "stdlib/hash/blake3s/blake3s.hpp" -#include "stdlib/hash/pedersen/pedersen.hpp" -#include "stdlib/hash/poseidon2/poseidon2.hpp" -#include "stdlib/merkle_tree/hash.hpp" -#include "stdlib/merkle_tree/membership.hpp" -#include "stdlib/merkle_tree/memory_store.hpp" -#include "stdlib/merkle_tree/memory_tree.hpp" -#include "stdlib/merkle_tree/merkle_tree.hpp" -// TODO(https://github.com/AztecProtocol/aztec-packages/issues/728): -// Consider moving nullifier tree logic out of barretenberg into aztec repo -#include "barretenberg/plonk/transcript/manifest.hpp" -#include "stdlib/merkle_tree/nullifier_tree/nullifier_leaf.hpp" -#include "stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp" -#include "stdlib/merkle_tree/nullifier_tree/nullifier_tree.hpp" -#include "stdlib/plonk_recursion/aggregation_state/aggregation_state.hpp" -#include "stdlib/plonk_recursion/aggregation_state/native_aggregation_state.hpp" -#include "stdlib/plonk_recursion/verification_key/verification_key.hpp" -#include "stdlib/plonk_recursion/verifier/program_settings.hpp" -#include "stdlib/plonk_recursion/verifier/verifier.hpp" -#include "stdlib/primitives/address/address.hpp" -#include "stdlib/primitives/bigfield/bigfield.hpp" -#include "stdlib/primitives/biggroup/biggroup.hpp" -#include "stdlib/primitives/bit_array/bit_array.hpp" -#include "stdlib/primitives/bool/bool.hpp" -#include "stdlib/primitives/byte_array/byte_array.hpp" -#include "stdlib/primitives/curves/bn254.hpp" -#include "stdlib/primitives/field/array.hpp" -#include "stdlib/primitives/field/field.hpp" -#include "stdlib/primitives/group/cycle_group.hpp" -#include "stdlib/primitives/packed_byte_array/packed_byte_array.hpp" -#include "stdlib/primitives/uint/uint.hpp" -#include "stdlib/primitives/witness/witness.hpp" diff --git a/barretenberg/cpp/src/barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget_impl.hpp b/barretenberg/cpp/src/barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget_impl.hpp index 50ae2dccefa..387ed72dc56 100644 --- a/barretenberg/cpp/src/barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget_impl.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget_impl.hpp @@ -66,7 +66,7 @@ void ProverPermutationWidget accumulators_ptrs[num_accumulators]; fr* accumulators[num_accumulators]; // Allocate the required number of length n scratch space arrays diff --git a/barretenberg/cpp/src/barretenberg/polynomials/polynomial_arithmetic.cpp b/barretenberg/cpp/src/barretenberg/polynomials/polynomial_arithmetic.cpp index 3530b816f3b..0694349f95d 100644 --- a/barretenberg/cpp/src/barretenberg/polynomials/polynomial_arithmetic.cpp +++ b/barretenberg/cpp/src/barretenberg/polynomials/polynomial_arithmetic.cpp @@ -1120,38 +1120,36 @@ bb::polynomial_arithmetic::LagrangeEvaluations get_lagrange_evaluations( // L_i(z) = L_1(Ê“.ω^{1-i}) = ------------------ // n.(Ê“.ω^{1-i)} - 1) // -template - requires SupportsFFT -Fr compute_barycentric_evaluation(const Fr* coeffs, +fr compute_barycentric_evaluation(const fr* coeffs, const size_t num_coeffs, - const Fr& z, - const EvaluationDomain& domain) + const fr& z, + const EvaluationDomain& domain) { - Fr* denominators = static_cast(aligned_alloc(64, sizeof(Fr) * num_coeffs)); + fr* denominators = static_cast(aligned_alloc(64, sizeof(fr) * num_coeffs)); - Fr numerator = z; + fr numerator = z; for (size_t i = 0; i < domain.log2_size; ++i) { numerator.self_sqr(); } - numerator -= Fr::one(); + numerator -= fr::one(); numerator *= domain.domain_inverse; // (Ê“^n - 1) / n - denominators[0] = z - Fr::one(); - Fr work_root = domain.root_inverse; // ω^{-1} + denominators[0] = z - fr::one(); + fr work_root = domain.root_inverse; // ω^{-1} for (size_t i = 1; i < num_coeffs; ++i) { denominators[i] = work_root * z; // denominators[i] will correspond to L_[i+1] (since our 'commented maths' notation indexes // L_i from 1). So Ê“.ω^{-i} = Ê“.ω^{1-(i+1)} is correct for L_{i+1}. - denominators[i] -= Fr::one(); // Ê“.ω^{-i} - 1 + denominators[i] -= fr::one(); // Ê“.ω^{-i} - 1 work_root *= domain.root_inverse; } - Fr::batch_invert(denominators, num_coeffs); + fr::batch_invert(denominators, num_coeffs); - Fr result = Fr::zero(); + fr result = fr::zero(); for (size_t i = 0; i < num_coeffs; ++i) { - Fr temp = coeffs[i] * denominators[i]; // f_i * 1/(Ê“.ω^{-i} - 1) + fr temp = coeffs[i] * denominators[i]; // f_i * 1/(Ê“.ω^{-i} - 1) result = result + temp; } @@ -1401,7 +1399,6 @@ template void divide_by_pseudo_vanishing_polynomial(std::vector, const size_t); template fr compute_kate_opening_coefficients(const fr*, fr*, const fr&, const size_t); template LagrangeEvaluations get_lagrange_evaluations(const fr&, const EvaluationDomain&, const size_t); -template fr compute_barycentric_evaluation(const fr*, const size_t, const fr&, const EvaluationDomain&); template void compress_fft(const fr*, fr*, const size_t, const size_t); template fr evaluate_from_fft(const fr*, const EvaluationDomain&, const fr&, const EvaluationDomain&); template fr compute_sum(const fr*, const size_t); diff --git a/barretenberg/cpp/src/barretenberg/polynomials/polynomial_arithmetic.hpp b/barretenberg/cpp/src/barretenberg/polynomials/polynomial_arithmetic.hpp index 61ee093e698..4de39d5a918 100644 --- a/barretenberg/cpp/src/barretenberg/polynomials/polynomial_arithmetic.hpp +++ b/barretenberg/cpp/src/barretenberg/polynomials/polynomial_arithmetic.hpp @@ -156,11 +156,10 @@ template LagrangeEvaluations get_lagrange_evaluations(const Fr& z, const EvaluationDomain& domain, const size_t num_roots_cut_out_of_vanishing_polynomial = 4); -template -Fr compute_barycentric_evaluation(const Fr* coeffs, - const size_t num_coeffs, - const Fr& z, - const EvaluationDomain& domain); +fr compute_barycentric_evaluation(const fr* coeffs, + unsigned long num_coeffs, + const fr& z, + const EvaluationDomain& domain); // Convert an fft with `current_size` point evaluations, to one with `current_size >> compress_factor` point evaluations template requires SupportsFFT diff --git a/barretenberg/cpp/srs_db/Earthfile b/barretenberg/cpp/srs_db/Earthfile new file mode 100644 index 00000000000..36688dac614 --- /dev/null +++ b/barretenberg/cpp/srs_db/Earthfile @@ -0,0 +1,19 @@ +VERSION 0.8 +FROM ubuntu:lunar + +RUN apt-get update && apt-get install -y curl + +build: + WORKDIR /build + COPY ./*.sh . + RUN ./download_ignition.sh 3 + RUN ./download_grumpkin.sh + # export srs-db for runners + SAVE ARTIFACT ignition ignition + SAVE ARTIFACT ignition grumpkin + +build-local: + # copy files locally + FROM +download + SAVE ARTIFACT ignition AS LOCAL ignition + SAVE ARTIFACT grumpkin AS LOCAL grumpkin diff --git a/barretenberg/cpp/srs_db/download_grumpkin.sh b/barretenberg/cpp/srs_db/download_grumpkin.sh index fb59a1ec806..56297198d27 100755 --- a/barretenberg/cpp/srs_db/download_grumpkin.sh +++ b/barretenberg/cpp/srs_db/download_grumpkin.sh @@ -1,11 +1,8 @@ #!/bin/sh -# TODO(https://github.com/AztecProtocol/barretenberg/issues/813) We don't *actually* download grumpkin yet. -# this just generates grumpkin points and links in a place where run_acir_tests.sh expects it. -# The above issue tracks the final pieces here. +# TODO(https://github.com/AztecProtocol/barretenberg/issues/898): Grumpkin needs to match new layout. set -eu - -# Enter build directory sibling to our script folder. -cd $(dirname $0)/../build -./bin/grumpkin_srs_gen 1048576 -mkdir -p ~/.bb-crs +# Enter script directory. +cd $(dirname $0) +./download_srs.sh "TEST%20GRUMPKIN" grumpkin/monomial 1 $@ +mkdir -p ~/.bb-crs ln -s ../srs_db/grumpkin/monomial ~/.bb-crs/monomial \ No newline at end of file diff --git a/barretenberg/cpp/srs_db/download_ignition.sh b/barretenberg/cpp/srs_db/download_ignition.sh index 4b0df0ee9fc..4e469d7505f 100755 --- a/barretenberg/cpp/srs_db/download_ignition.sh +++ b/barretenberg/cpp/srs_db/download_ignition.sh @@ -1,68 +1,5 @@ #!/bin/sh -# Downloads the ignition trusted setup transcripts. -# -# See here for details of the contents of the transcript.dat files: -# https://github.com/AztecProtocol/ignition-verification/blob/master/Transcript_spec.md -# -# To download all transcripts. -# ./download_ignition.sh -# -# To download a range of transcripts, e.g. 0, 1 and 2. -# ./download_ignition.sh 2 -# -# If a checksums file is available, it will be used to validate if a download is required -# and also check the validity of the downloaded transcripts. If not the script downloads -# whatever is requested but does not check the validity of the downloads. set -eu - # Enter script directory. cd $(dirname $0) - -mkdir -p ignition -cd ignition -mkdir -p monomial -cd monomial -NUM=${1:-19} -RANGE_START=${2:-} -RANGE_END=${3:-} -APPEND=${4:-"false"} - -if command -v sha256sum > /dev/null; then - SHASUM=sha256sum -else - SHASUM="shasum -a 256" -fi - -checksum() { - grep transcript${1}.dat checksums | $SHASUM -c - return $? -} - -download() { - # Initialize an empty variable for the Range header - RANGE_HEADER="" - - # If both RANGE_START and RANGE_END are set, add them to the Range header - if [ -n "$RANGE_START" ] && [ -n "$RANGE_END" ]; then - RANGE_HEADER="-H Range:bytes=$RANGE_START-$RANGE_END" - fi - - # Download the file - if [ "$APPEND" = "true" ]; then - curl $RANGE_HEADER https://aztec-ignition.s3-eu-west-2.amazonaws.com/MAIN%20IGNITION/monomial/transcript${1}.dat >> transcript${1}.dat - else - curl $RANGE_HEADER https://aztec-ignition.s3-eu-west-2.amazonaws.com/MAIN%20IGNITION/monomial/transcript${1}.dat > transcript${1}.dat - fi - -} - -for TRANSCRIPT in $(seq 0 $NUM); do - NUM=$(printf %02d $TRANSCRIPT) - if [ -f checksums ] && [ -z "$RANGE_START" ] && [ -z "$RANGE_END" ] ; then - checksum $NUM && continue - download $NUM - checksum $NUM || exit 1 - else - download $NUM - fi -done +./download_srs.sh "MAIN%20IGNITION" ignition/monomial $@ \ No newline at end of file diff --git a/barretenberg/cpp/srs_db/download_srs.sh b/barretenberg/cpp/srs_db/download_srs.sh new file mode 100755 index 00000000000..df7ffd03041 --- /dev/null +++ b/barretenberg/cpp/srs_db/download_srs.sh @@ -0,0 +1,52 @@ +#!/bin/sh +set -eu + +AWS_BUCKET=$1 +DESTINATION=$2 +NUM=${3:-19} +RANGE_START=${4:-} +RANGE_END=${5:-} +APPEND=${6:-"false"} + +mkdir -p "$DESTINATION" +cd "$DESTINATION" + +if command -v sha256sum > /dev/null; then + SHASUM=sha256sum +else + SHASUM="shasum -a 256" +fi + +checksum() { + grep transcript${1}.dat checksums | $SHASUM -c + return $? +} + +download() { + # Initialize an empty variable for the Range header + RANGE_HEADER="" + + # If both RANGE_START and RANGE_END are set, add them to the Range header + if [ -n "$RANGE_START" ] && [ -n "$RANGE_END" ]; then + RANGE_HEADER="-H Range:bytes=$RANGE_START-$RANGE_END" + fi + + # Download the file + if [ "$APPEND" = "true" ]; then + curl $RANGE_HEADER https://aztec-ignition.s3-eu-west-2.amazonaws.com/$AWS_BUCKET/monomial/transcript${1}.dat >> transcript${1}.dat + else + curl $RANGE_HEADER https://aztec-ignition.s3-eu-west-2.amazonaws.com/$AWS_BUCKET/monomial/transcript${1}.dat > transcript${1}.dat + fi + +} + +for TRANSCRIPT in $(seq 0 $NUM); do + NUM=$(printf %02d $TRANSCRIPT) + if [ -f checksums ] && [ -z "$RANGE_START" ] && [ -z "$RANGE_END" ] ; then + checksum $NUM && continue + download $NUM + checksum $NUM || exit 1 + else + download $NUM + fi +done diff --git a/barretenberg/cpp/srs_db/grumpkin/monomial/README.md b/barretenberg/cpp/srs_db/grumpkin/monomial/README.md deleted file mode 100644 index 9d3e53e8894..00000000000 --- a/barretenberg/cpp/srs_db/grumpkin/monomial/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# Quick-and-dirty Grumpkin transcript - -The Grumpkin transcript currently departs in structure from the BN254 -transcript in that: - - It does not contain a checksum - - It does not contain any g2 points (indeed, there is no grumpkin::g2). - - The transcript generation binary only produces a single transcript file. - If more than 504000 points are desired at some point, it is likely we will - need to a small refactor so that more files are created. - -A full-length transcript file containing 504000 points would have -BN254 transcript00.dat size: 322560412 -Grumpkin transcript00.dat size: 322560028 -322560028 - 322560412 = 384 = 256 + 128 - ^^^^^^^^^^^ ^^^^^^^^ - 2 g2 points checksum diff --git a/barretenberg/cpp/srs_db/ignition/monomial/checksums b/barretenberg/cpp/srs_db/ignition/monomial/checksums deleted file mode 100644 index 177b9441eb1..00000000000 --- a/barretenberg/cpp/srs_db/ignition/monomial/checksums +++ /dev/null @@ -1,21 +0,0 @@ -227512f4233e1b068e5dbfcf1a7fbb8b03d18050844eec29cb34af283fa6a7c9 g2.dat -0f94b39426f5a57d1a17742624f7d3170a32e19d6c5e16f202281439dbe48695 transcript00.dat -fe8919733b0ef279d31787f2c80e67a014dc50ad2790640157a7d411307784a3 transcript01.dat -0602f52fa0dd81a26cc5e1db5c068caf49118c30921238f41d8d16169041e15f transcript02.dat -b02f5c2b6d6017f90f3389ba2796465a11f582b49bbc900e76583a64bc8b32a5 transcript03.dat -66c14ce89a64345f70ac06f11bd327ce9b7aa3a33a7f27c8f094ba8a54374032 transcript04.dat -587bda8a743a4c21ff39bfbf269ed6f670ad2001e2490f001dd4ac6244d717ad transcript05.dat -08bddb87d7a0b063ff4e57e2a29d356398d2730fe33dddb54043c7e22015fe68 transcript06.dat -b3d36da80088d763490f752744036fc51b76ce623599926e72c96fac1a17ef0c transcript07.dat -c3f91827a2c0a187a77a01901371f8ca70731d3efbb1488135081318f86159bc transcript08.dat -98f98df7536245dcd1b40c12b0aea5f28a7fa6de9f4c7621164099fb57c9be64 transcript09.dat -8860f593a612f77f3ead23eefa201683b2576882365821e6fdcf40eab6ab20c6 transcript10.dat -b16a20941f3fe56f05f9df1fda79f5c43dc9f977ea0bcac9731431efafd3262c transcript11.dat -803c0bb9a43025602ec6ed4ddd303667da8c2e16a94410a49d5c95286737b00b transcript12.dat -c74beaa60e92d085a46b4f2336d452f4ba193a5304cef3a928eae7c87479b533 transcript13.dat -fc8a919e0629441dc1bce2e913aa87253a543cd343e67e00451bb98cecf9b701 transcript14.dat -1fa0c3f39221e83c205d081d7e09367e7a989323d0e6435232bc8f662491188f transcript15.dat -0e81089628ae03b869594dc4b170194f7f6fbfdecf6c9566f409b85697e8d5be transcript16.dat -82415a8e8a5c6106f199fddf0319151566a857fa2b11e9fe9c5d7568e5fc7612 transcript17.dat -099c74b4e257bd146323a161259b733d3c9c7702124cf52f620dc9f66ecaa11c transcript18.dat -dfa9e366e93c179325a969dc694986fa27392ba098e1941aeceae9f4683a3d66 transcript19.dat diff --git a/barretenberg/cpp/srs_db/ignition/monomial/g2.dat b/barretenberg/cpp/srs_db/ignition/monomial/g2.dat deleted file mode 100644 index 22bc78828b3..00000000000 --- a/barretenberg/cpp/srs_db/ignition/monomial/g2.dat +++ /dev/null @@ -1 +0,0 @@ -~#쓈ƒ°ŸYD;2‹¼‰µ³˜µ—NÄÕ¸7¼ÂNþ0úÀ“ƒÁêQØz5Ž‹çÿNX‘Þè&²QöñÇ…J‡ÔÚÌ^UæÝ?–æ΢VG[Båa^"þ½£ÀÀc*îA<€Új_äœò FAù›¤ÒQVÁ»šr…üci÷ã \ No newline at end of file diff --git a/barretenberg/ts/.earthlyignore b/barretenberg/ts/.earthlyignore new file mode 100644 index 00000000000..9981fca1804 --- /dev/null +++ b/barretenberg/ts/.earthlyignore @@ -0,0 +1 @@ +**/*.wasm \ No newline at end of file diff --git a/barretenberg/ts/Earthfile b/barretenberg/ts/Earthfile new file mode 100644 index 00000000000..744a7c7b750 --- /dev/null +++ b/barretenberg/ts/Earthfile @@ -0,0 +1,48 @@ +VERSION 0.8 + +FROM node:18.19.0 +WORKDIR /build + +# minimum files to download yarn packages +# keep timestamps for incremental builds +COPY --keep-ts --dir .yarn package.json yarn.lock .yarnrc.yml . +RUN yarn --immutable + +# other source files +COPY --keep-ts --dir src *.json *.js *.cjs . + +# copy over wasm build from cpp folder +COPY ../cpp/+preset-wasm/bin/barretenberg.wasm src/barretenberg_wasm/barretenberg-threads.wasm +# TODO for now there is no real single-threaded WASM. See if anyone hits problems. +COPY ../cpp/+preset-wasm/bin/barretenberg.wasm src/barretenberg_wasm/barretenberg.wasm +COPY ../cpp/+preset-wasm/bin/barretenberg.wasm dest/node/barretenberg_wasm/barretenberg-threads.wasm +COPY ../cpp/+preset-wasm/bin/barretenberg.wasm dest/node-cjs/barretenberg_wasm/barretenberg-threads.wasm + +esm: + RUN yarn build:esm + SAVE ARTIFACT /build + +cjs: + COPY --keep-ts scripts/cjs_postprocess.sh scripts/ + RUN yarn build:cjs + SAVE ARTIFACT /build + +browser: + RUN yarn build:browser + SAVE ARTIFACT /build + +test-prettier-format: + RUN yarn formatting + +build: + # collect all our build types + COPY +esm/build /build + COPY +cjs/build /build + COPY +browser/build /build + # We want to create a pure package, as would be published to npm, for consuming projects. + RUN yarn pack && tar zxf package.tgz && rm package.tgz && mv package ../ts + SAVE ARTIFACT /build + +test: + BUILD +test-prettier-format + RUN yarn test diff --git a/boxes/Earthfile b/boxes/Earthfile new file mode 100644 index 00000000000..c8a92975bb7 --- /dev/null +++ b/boxes/Earthfile @@ -0,0 +1,18 @@ +VERSION 0.8 +# Produces a container that can be run to test a specific box. See docker-compose.yml. + +build: + # We need yarn. Start fresh container. + FROM node:18.19.0 + RUN apt update && apt install netcat-openbsd + COPY ../yarn-project+build/build /build + COPY ../noir/+nargo/nargo /build/noir/noir-repo/target/release/nargo + COPY ../noir-projects/+build/aztec-nr /build/noir-projects/aztec-nr + COPY ../noir-projects/+build/noir-protocol-circuits/crates/types /build/noir-projects/noir-protocol-circuits/crates/types + WORKDIR /build/boxes + COPY . . + ENV AZTEC_NARGO=/build/noir/noir-repo/target/release/nargo + ENV AZTEC_CLI=/build/yarn-project/cli/aztec-cli-dest + RUN yarn && yarn build + RUN npx -y playwright@1.42 install --with-deps + ENTRYPOINT ["/bin/sh", "-c"] \ No newline at end of file diff --git a/build-system/scripts/remote_runner b/build-system/scripts/remote_initialize similarity index 96% rename from build-system/scripts/remote_runner rename to build-system/scripts/remote_initialize index 6283050283c..5a1ed5cf6f7 100755 --- a/build-system/scripts/remote_runner +++ b/build-system/scripts/remote_initialize @@ -23,5 +23,3 @@ echo "Git checkout completed." BASH_ENV=/tmp/bash_env echo "Calling setup env..." source ./build-system/scripts/setup_env "$COMMIT_HASH" "$COMMIT_TAG" "$JOB_NAME" "$GIT_REPOSITORY_URL" "$BRANCH" "$PULL_REQUEST" -echo "Calling $@..." -$@ diff --git a/build-system/scripts/remote_run_script b/build-system/scripts/remote_run_script index 855b3f14610..c157e4f9131 100755 --- a/build-system/scripts/remote_run_script +++ b/build-system/scripts/remote_run_script @@ -15,11 +15,11 @@ shift SSH_CONFIG_PATH=${SSH_CONFIG_PATH:-$BUILD_SYSTEM_PATH/remote/ssh_config} # Copy the runner script to spot instance. This is what we actually run. -echo "Copying ./remote_runner to $IP..." -scp -rF $SSH_CONFIG_PATH $BUILD_SYSTEM_PATH/scripts/remote_runner $IP:. +echo "Copying ./remote_initialize to $IP..." +scp -rF $SSH_CONFIG_PATH $BUILD_SYSTEM_PATH/scripts/remote_initialize $IP:. # Run script on remote instance, passing environment variables. -echo "Running ./remote_runner $@ on $IP..." +echo "Running $@ on $IP..." ssh -A -F $SSH_CONFIG_PATH $IP " export COMMIT_HASH=$COMMIT_HASH export COMMIT_TAG=$COMMIT_TAG @@ -33,5 +33,6 @@ ssh -A -F $SSH_CONFIG_PATH $IP " # temp while we transitioning to avm export AVM_ENABLED=${AVM_ENABLED:-} - ./remote_runner $@ + source ./remote_initialize + $@ " diff --git a/build-system/scripts/setup_env b/build-system/scripts/setup_env index 63c7035e169..7280711ed1b 100755 --- a/build-system/scripts/setup_env +++ b/build-system/scripts/setup_env @@ -132,6 +132,7 @@ if [ -n "$COMMIT_HASH" ]; then mkdir -p ~/.ssh echo ${BUILD_INSTANCE_KEY:-} | base64 -d > ~/.ssh/build_instance_key chmod 600 ~/.ssh/build_instance_key + cp $BUILD_SYSTEM_PATH/remote/ssh_config ~/.ssh/config if [[ "$COMMIT_MESSAGE" == *"[ci debug]"* ]]; then echo export BUILD_SYSTEM_DEBUG=1 >> $BASH_ENV diff --git a/foundry/Dockerfile b/foundry/Dockerfile new file mode 100644 index 00000000000..65171755174 --- /dev/null +++ b/foundry/Dockerfile @@ -0,0 +1,35 @@ +FROM alpine:3.18 as build-environment + +ARG TARGETARCH +WORKDIR /opt + +RUN apk add clang lld curl build-base linux-headers git \ + && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > rustup.sh \ + && chmod +x ./rustup.sh \ + && ./rustup.sh -y + +RUN [[ "$TARGETARCH" = "arm64" ]] && echo "export CFLAGS=-mno-outline-atomics" >> $HOME/.profile || true + +WORKDIR /opt +RUN git clone --depth 1 --branch nightly-de33b6af53005037b463318d2628b5cfcaf39916 https://github.com/foundry-rs/foundry.git +WORKDIR /opt/foundry + +RUN source $HOME/.profile && cargo build --release \ + && mkdir out \ + && mv target/release/forge out/forge \ + && mv target/release/cast out/cast \ + && mv target/release/anvil out/anvil \ + && mv target/release/chisel out/chisel \ + && strip out/forge \ + && strip out/cast \ + && strip out/chisel \ + && strip out/anvil; + +FROM alpine:3.18 as foundry-client +RUN apk add --no-cache linux-headers git +COPY --from=build-environment /opt/foundry/out/forge /usr/local/bin/forge +COPY --from=build-environment /opt/foundry/out/cast /usr/local/bin/cast +COPY --from=build-environment /opt/foundry/out/anvil /usr/local/bin/anvil +COPY --from=build-environment /opt/foundry/out/chisel /usr/local/bin/chisel +RUN adduser -Du 1000 foundry +ENTRYPOINT ["/bin/sh", "-c"] \ No newline at end of file diff --git a/foundry/Earthfile b/foundry/Earthfile new file mode 100644 index 00000000000..24c8bf5512a --- /dev/null +++ b/foundry/Earthfile @@ -0,0 +1,11 @@ +VERSION 0.8 + +build: + ARG TARGETARCH + FROM DOCKERFILE . + SAVE IMAGE --push aztecprotocol/cache:foundry-nightly-de33b6af53005037b463318d2628b5cfcaf39916-$TARGETARCH + +get: + ARG TARGETARCH + # If this is failing, we need to run earthly --push +build + FROM aztecprotocol/cache:foundry-nightly-de33b6af53005037b463318d2628b5cfcaf39916-$TARGETARCH diff --git a/l1-contracts/Earthfile b/l1-contracts/Earthfile new file mode 100644 index 00000000000..7383b952ab2 --- /dev/null +++ b/l1-contracts/Earthfile @@ -0,0 +1,23 @@ +VERSION 0.8 +FROM ubuntu:lunar +RUN apt update && apt install curl git jq bash nodejs npm python3.11-full python3-pip -y + +# Use virtualenv, do not try to use pipx, it's not working. +RUN python3 -m venv /root/.venv +RUN /root/.venv/bin/pip3 install slither-analyzer==0.10.0 slitherin==0.5.0 +RUN curl -L https://foundry.paradigm.xyz | bash + +# Set env variables for foundry and venv +ENV PATH="${PATH}:/root/.foundry/bin:/root/.venv/bin" +RUN foundryup + +# Install yarn and solhint. +RUN npm install --global yarn solhint + +WORKDIR /build +COPY --keep-ts --dir lib scripts src terraform test *.json *.toml *.sh . + +build: + RUN git init && git add . && yarn lint && yarn slither && yarn slither-has-diff + RUN forge clean && forge fmt --check && forge build && forge test + SAVE ARTIFACT out diff --git a/noir-projects/.earthlyignore b/noir-projects/.earthlyignore new file mode 100644 index 00000000000..f2a4093411b --- /dev/null +++ b/noir-projects/.earthlyignore @@ -0,0 +1 @@ +**/target \ No newline at end of file diff --git a/noir-projects/Earthfile b/noir-projects/Earthfile new file mode 100644 index 00000000000..85673639c6b --- /dev/null +++ b/noir-projects/Earthfile @@ -0,0 +1,24 @@ +VERSION 0.8 +FROM ubuntu:lunar + +# Install nargo +COPY ../noir/+nargo/nargo /usr/bin/nargo +# Install transpiler +COPY ../avm-transpiler/+build/avm-transpiler /usr/bin/avm-transpiler + +WORKDIR /build +# Copy source. +COPY --dir aztec-nr noir-contracts noir-protocol-circuits . + +build: + RUN cd noir-contracts && NARGO=nargo TRANSPILER=avm-transpiler ./bootstrap.sh + RUN cd noir-protocol-circuits && NARGO=nargo ./bootstrap.sh + SAVE ARTIFACT aztec-nr + SAVE ARTIFACT noir-contracts + SAVE ARTIFACT noir-protocol-circuits + +test: + FROM +build + RUN cd noir-protocol-circuits && nargo test --silence-warnings + RUN cd aztec-nr && nargo test --silence-warnings + RUN cd noir-contracts && nargo test --silence-warnings diff --git a/noir-projects/noir-contracts/bootstrap.sh b/noir-projects/noir-contracts/bootstrap.sh index 06c1df51bfb..eccec0ff99e 100755 --- a/noir-projects/noir-contracts/bootstrap.sh +++ b/noir-projects/noir-contracts/bootstrap.sh @@ -16,10 +16,12 @@ if [ -n "$CMD" ]; then fi echo "Compiling contracts..." -../../noir/noir-repo/target/release/nargo compile --silence-warnings +NARGO=${NARGO:-../../noir/noir-repo/target/release/nargo} +$NARGO compile --silence-warnings echo "Transpiling avm contracts..." for contract_json in target/avm_test_*.json; do echo Transpiling $contract_json... - ../../avm-transpiler/target/release/avm-transpiler $contract_json $contract_json + TRANSPILER=${TRANSPILER:-../../avm-transpiler/target/release/avm-transpiler} + $TRANSPILER $contract_json $contract_json done \ No newline at end of file diff --git a/noir-projects/noir-protocol-circuits/bootstrap.sh b/noir-projects/noir-protocol-circuits/bootstrap.sh index b7af2ad14f6..e647f07b80f 100755 --- a/noir-projects/noir-protocol-circuits/bootstrap.sh +++ b/noir-projects/noir-protocol-circuits/bootstrap.sh @@ -16,4 +16,5 @@ if [ -n "$CMD" ]; then fi echo "Compiling protocol circuits..." -../../noir/noir-repo/target/release/nargo compile --silence-warnings \ No newline at end of file +NARGO=${NARGO:-../../noir/noir-repo/target/release/nargo} +$NARGO compile --silence-warnings \ No newline at end of file diff --git a/noir/Earthfile b/noir/Earthfile new file mode 100644 index 00000000000..f2d91a4cd7b --- /dev/null +++ b/noir/Earthfile @@ -0,0 +1,111 @@ +VERSION 0.8 + + +nargo: + FROM rust:bullseye + RUN apt update && apt install -y libc++1 + WORKDIR /build + # Relevant source (TODO finer-grained 'tooling') + COPY --keep-ts --dir \ + noir-repo/acvm-repo \ + noir-repo/aztec_macros \ + noir-repo/compiler \ + noir-repo/noir_stdlib \ + noir-repo/tooling \ + noir-repo/test_programs \ + noir-repo/Cargo.lock \ + noir-repo/Cargo.toml \ + noir-repo + + # TODO(AD) is this OK as a content hash? + ENV COMMIT_HASH=$(find . -type f -exec sha256sum {} ';' | sort | sha256sum | awk '{print $1}') + COPY --keep-ts ./scripts/bootstrap_native.sh ./scripts/bootstrap_native.sh + RUN ./scripts/bootstrap_native.sh + SAVE ARTIFACT /build/noir-repo/target/release/nargo nargo + SAVE ARTIFACT /build/noir-repo/target/release/acvm acvm + +packages: + FROM node:20 + RUN curl https://sh.rustup.rs -sSf | bash -s -- -y + RUN echo 'source $HOME/.cargo/env' >> $HOME/.bashrc + RUN apt update && apt install -y jq libc++1 + WORKDIR /build + + # Relevant source (TODO finer-grained) + COPY --keep-ts --dir \ + noir-repo/acvm-repo \ + noir-repo/aztec_macros \ + noir-repo/compiler \ + noir-repo/docs \ + noir-repo/noir_stdlib \ + noir-repo/scripts \ + noir-repo/test_programs \ + noir-repo/tooling \ + noir-repo/Cargo.lock \ + noir-repo/.yarnrc.yml \ + noir-repo/.yarn \ + noir-repo/yarn.lock \ + noir-repo/package.json \ + noir-repo/LICENSE* \ + noir-repo/*.toml \ + noir-repo/*.json \ + noir-repo/*.js \ + noir-repo/.github \ + noir-repo/.envrc \ + noir-repo + + COPY --keep-ts noir-repo/.github/scripts noir-repo/.github/scripts + COPY --keep-ts ./scripts/bootstrap_packages.sh ./scripts/bootstrap_packages.sh + # TODO(AD) is this OK as a content hash? + ENV COMMIT_HASH=$(find . -type f -exec sha256sum {} ';' | sort | sha256sum | awk '{print $1}') + RUN PATH="/root/.cargo/bin:$PATH" ./scripts/bootstrap_packages.sh + SAVE ARTIFACT packages + +run: + # When running the container, mount the users home directory to same location. + FROM ubuntu:lunar + # Install Tini as nargo doesn't handle signals properly. + # Install git as nargo needs it to clone. + RUN apt-get update && apt-get install -y git tini && rm -rf /var/lib/apt/lists/* && apt-get clean + COPY +build/. /build + ENTRYPOINT ["/usr/bin/tini", "--", "/build/nargo"] + +build: + BUILD +nargo + BUILD +packages + +# TOOD +# test-packages +# FROM aztecprotocol/noir AS noir + +# FROM node:20 AS builder +# COPY --from=noir /usr/src/noir/noir-repo/target/release /usr/src/noir/noir-repo/target/release +# ENV PATH=${PATH}:/usr/src/noir/noir-repo/target/release +# RUN curl https://sh.rustup.rs -sSf | bash -s -- -y +# RUN echo 'source $HOME/.cargo/env' >> $HOME/.bashrc +# ENV PATH=/root/.cargo/bin:${PATH} +# RUN apt update && apt install -y jq libc++1 +# ARG COMMIT_HASH +# ENV COMMIT_HASH=${COMMIT_HASH} + +# WORKDIR /usr/src/noir +# COPY . . +# RUN ./scripts/test_js_packages.sh + +# # Don't waste time pushing a huge container back to ECR as nothing needs the output. +# FROM scratch +# COPY --from=builder /usr/src/noir/README.md /usr/src/noir/README.md + +# TODO +# test: +# FROM rust:bullseye +# ARG COMMIT_HASH +# ENV COMMIT_HASH=${COMMIT_HASH} +# RUN apt update && apt install -y libc++1 +# WORKDIR /usr/src/noir +# COPY . . +# RUN ./scripts/test_native.sh + +# # Don't waste time pushing a huge container back to ECR as nothing needs the output. +# FROM scratch +# COPY --from=0 /usr/src/noir/README.md /usr/src/noir/README.md diff --git a/noir/noir-repo/.github/scripts/wasm-bindgen-install.sh b/noir/noir-repo/.github/scripts/wasm-bindgen-install.sh index a548372ee2c..20908003693 100755 --- a/noir/noir-repo/.github/scripts/wasm-bindgen-install.sh +++ b/noir/noir-repo/.github/scripts/wasm-bindgen-install.sh @@ -6,7 +6,7 @@ cd $(dirname "$0") ./cargo-binstall-install.sh # Install wasm-bindgen-cli. -if [ "$(wasm-bindgen --version | cut -d' ' -f2)" != "0.2.86" ]; then +if [ "$(wasm-bindgen --version &> /dev/null | cut -d' ' -f2)" != "0.2.86" ]; then echo "Building wasm-bindgen..." cargo binstall wasm-bindgen-cli@0.2.86 --force --no-confirm fi diff --git a/scripts/earthly b/scripts/earthly new file mode 100755 index 00000000000..ff0137b8415 --- /dev/null +++ b/scripts/earthly @@ -0,0 +1,31 @@ + +#!/usr/bin/env bash +[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace + +# Aztec build/test/bench tool +# Thin wrapper for earthly that helps with building targets. +# Adds autodownloading earthly and timing code. +# Usage: +# Go to folder, e.g. docs, use az to build +# To run a target in the Earthfile, use 'az +'. +# Spports all commands 'earthly' can take. + +set -euo pipefail + +mkdir -p $(dirname $0)/.earthly + +EARTHLY=$(dirname $0)/.earthly/earthly +EARTHLY_CONFIG=$(dirname $0)/earthly-config.yml + +if ! [ -f "$EARTHLY" ] ; then + wget https://github.com/earthly/earthly/releases/latest/download/earthly-linux-amd64 -O "$EARTHLY"; + chmod +x "$EARTHLY" +fi + +EARTHLY_FLAGS="-P --disable-remote-registry-proxy --use-inline-cache --save-inline-cache" +if [ $# -eq 0 ]; then + # By default, try for a +build target + "$EARTHLY" $EARTHLY_FLAGS +build +else + "$EARTHLY" $EARTHLY_FLAGS $@ +fi \ No newline at end of file diff --git a/scripts/earthly-config.yml b/scripts/earthly-config.yml new file mode 100644 index 00000000000..c2b6747834f --- /dev/null +++ b/scripts/earthly-config.yml @@ -0,0 +1,3 @@ +global: + cache_size_mb: 100000 + container_frontend: docker-shell diff --git a/scripts/earthly-timed b/scripts/earthly-timed new file mode 100755 index 00000000000..62ad856b734 --- /dev/null +++ b/scripts/earthly-timed @@ -0,0 +1,55 @@ + +#!/usr/bin/env bash +[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace + +# Aztec build/test/bench tool +# Thin wrapper for earthly that helps with building targets. +# Adds autodownloading earthly and timing code. +# Usage: +# Go to folder, e.g. docs, use az to build +# To run a target in the Earthfile, use 'az +'. +# Spports all commands 'earthly' can take. + +set -euo pipefail + +export FORCE_COLOR=1 +add_timestamps() { + gray=$(tput setaf 8) + normal=$(tput sgr0) + while IFS= read -r line; do printf "${gray}%(%H:%M:%S)T${normal} " ; echo "$line" ; done +} +report_time() { + end_time=$(date +%s) + duration=$((end_time - start_time)) + gray=$(tput setaf 8) + normal=$(tput sgr0) + local hours=$((duration / 3600)) + local minutes=$(( (duration % 3600) / 60 )) + local seconds=$((duration % 60)) + + printf "${gray}" + if (( hours > 0 )); then + printf "%d hours, %d minutes, and %d seconds" $hours $minutes $seconds + elif (( minutes > 0 )); then + printf "%d minutes and %d seconds" $minutes $seconds + else + printf "%d seconds" $seconds + fi + printf "${normal}\n" +} +# on finish +trap report_time EXIT +start_time=$(date +%s) + +INTERACTIVE=${INTERACTIVE:-false} +for arg in "$@"; do + if [ "$arg" = "-i" ] || [ "$arg" = "--interactive" ]; then + INTERACTIVE=true + break + fi +done +if [ $INTERACTIVE = true ] ; then + $(dirname $0)/earthly $@ +else + $(dirname $0)/earthly $@ 2>&1 | add_timestamps >&2 +fi \ No newline at end of file diff --git a/yarn-project/.earthlyignore b/yarn-project/.earthlyignore new file mode 100644 index 00000000000..25434442e60 --- /dev/null +++ b/yarn-project/.earthlyignore @@ -0,0 +1,56 @@ +# Must include the .gitignore for all child projects as this is used by Earthly +# Note due to how we use Eartlhy each .gitignore MUST accompany any earthfile that might actually copy these artifacts +**/Earthfile +**/Readme.md +**/Dockerfile* +**/docker-compose*.yml + +# .gitignore contents: +**/dest +**/node_modules +**/.cache +.env* +*.log +*.swp +**/.tsbuildinfo +**/tsconfig.tsbuildinfo +**/.eslintcache +**/target +accounts/src/artifacts +aztec-faucet/data* +aztec-node/data* +aztec-js/src/account_contract/artifacts +aztec/log +circuits.js/fixtures/*.json +docs/dist +end-to-end/addresses.json +end-to-end/log +end-to-end/data +end-to-end/src/web/main.js +end-to-end/src/web/main.js.LICENSE.txt +entry-points/src/artifacts +l1-contracts/generated +noir-compiler/target/ +noir-compiler/proofs/ +noir-compiler/Prover.toml +noir-compiler/Verifier.toml +noir-compiler/noir-protocol-circuits-types/.gitignore wow +noir-compiler/proofs/ +noir-compiler/Prover.toml +noir-compiler/Verifier.toml +noir-compiler/src/target +noir-compiler/src/crs +noir-compiler/src/types +protocol-contracts/src/artifacts +scripts/tmp +noir-contracts.js/src +noir-contracts.js/artifacts/ +noir-contracts.js/codegenCache.json + + +.yarn/* +!.yarn/patches +!.yarn/plugins +!.yarn/releases +!.yarn/sdks +!.yarn/versions diff --git a/yarn-project/.gitignore b/yarn-project/.gitignore index a8ce42c3200..1db6722f9c9 100644 --- a/yarn-project/.gitignore +++ b/yarn-project/.gitignore @@ -1,15 +1,46 @@ -dest -node_modules -.cache +# To be maintained along with .earthlyinore +**/dest +**/node_modules +**/.cache .env* *.log *.swp -.tsbuildinfo -tsconfig.tsbuildinfo -.eslintcache -simulator/target -.debounce-* -.tsc.pid +**/.tsbuildinfo +**/tsconfig.tsbuildinfo +**/.eslintcache +**/target +**/.debounce-* +**/.tsc.pid +accounts/src/artifacts +aztec-faucet/data* +aztec-node/data* +aztec-js/src/account_contract/artifacts +aztec/log +circuits.js/fixtures/*.json +docs/dist +end-to-end/addresses.json +end-to-end/log +end-to-end/data +end-to-end/src/web/main.js +end-to-end/src/web/main.js.LICENSE.txt +entry-points/src/artifacts +l1-contracts/generated +noir-compiler/target/ +noir-compiler/proofs/ +noir-compiler/Prover.toml +noir-compiler/Verifier.toml +noir-compiler/noir-protocol-circuits-types/.gitignore wow +noir-compiler/proofs/ +noir-compiler/Prover.toml +noir-compiler/Verifier.toml +noir-compiler/src/target +noir-compiler/src/crs +noir-compiler/src/types +protocol-contracts/src/artifacts +scripts/tmp +noir-contracts.js/src +noir-contracts.js/artifacts/ +noir-contracts.js/codegenCache.json .yarn/* !.yarn/patches diff --git a/yarn-project/Earthfile b/yarn-project/Earthfile new file mode 100644 index 00000000000..d2dba999259 --- /dev/null +++ b/yarn-project/Earthfile @@ -0,0 +1,114 @@ +VERSION 0.8 +FROM node:18.19.0 +RUN apt update && apt install -y jq curl perl && rm -rf /var/lib/apt/lists/* && apt-get clean + +# copy bb-js and noir-packages +COPY ../barretenberg/ts/+build/build /build/barretenberg/ts +COPY ../noir/+packages/packages /build/noir/packages +# install acvm binary +COPY ../noir/+nargo/acvm /usr/bin/acvm +COPY --dir ../noir-projects/+build/. /build/noir-projects +COPY ../l1-contracts/+build/out /build/l1-contracts/out + +WORKDIR /build/yarn-project +# copy source +COPY --keep-ts --dir * *.json .yarn .yarnrc.yml . + +# We install a symlink to yarn-project's node_modules at a location that all portalled packages can find as they +# walk up the tree as part of module resolution. The supposedly idiomatic way of supporting module resolution +# correctly for portalled packages, is to use --preserve-symlinks when running node. +# This does kind of work, but jest doesn't honor it correctly, so this seems like a neat workaround. +# Also, --preserve-symlinks causes duplication of portalled instances such as bb.js, and breaks the singleton logic +# by initialising the module more than once. So at present I don't see a viable alternative. +RUN ln -s /build/yarn-project/node_modules /build/node_modules + +# Target for main build process +build: + ARG EARTHLY_CI + # TODO: Replace puppeteer with puppeteer-core to avoid this. + # TODO encapsulate in bash script for cleanliness + ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true + IF $EARTHLY_CI # Don't cache CI builds. + RUN ./bootstrap.sh && yarn workspaces focus @aztec/cli @aztec/aztec --production && yarn cache clean + ELSE + ENV YARN_CACHE_FOLDER /build/yarn-cache + # Use a mount for incremental builds locally. + RUN --mount type=cache,target=/build/node_modules_cache --mount type=cache,target=/build/yarn-cache \ + mkdir -p node_modules_cache && \ + mv node_modules_cache node_modules && \ + ./bootstrap.sh && \ + yarn workspaces focus @aztec/cli @aztec/aztec --production && \ + cp -r node_modules node_modules_cache + END + SAVE ARTIFACT /build + +# TODO versioning flow at end before publish? +# ENV COMMIT_TAG=$EARTHLY_BUILD_SHA +# RUN ./scripts/version_packages.sh + +# run: +# FROM node:18.19.1-slim +# ARG COMMIT_TAG="" +# ENV COMMIT_TAG=$COMMIT_TAG +# COPY --from=builder /build /build +# WORKDIR /build/yarn-project +# ENTRYPOINT ["yarn"] + +aztec: + FROM +build + # ENV vars for using native ACVM simulation + ENV ACVM_BINARY_PATH="/usr/bin/acvm" ACVM_WORKING_DIRECTORY="/tmp/acvm" + ENTRYPOINT ["node", "--no-warnings", "/build/yarn-project/aztec/dest/bin/index.js"] + EXPOSE 8080 + # TODO(AD) the following are the biggest node modules bundled, should they be deleted as they are build tools? + # 25840 @jest + # 31544 typescript + # 62368 @types + + # The version has been updated in yarn-project. + # Adding COMMIT_TAG here to rebuild versioned image. + ARG COMMIT_TAG="" + +end-to-end: + # compilation artifacts for end-to-end + # TODO encapsulate in bash script for cleanliness + ARG EARTHLY_CI + IF $EARTHLY_CI # Don't cache CI builds. + RUN ./bootstrap.sh && \ + yarn workspace @aztec/end-to-end run build:web && \ + yarn workspaces focus @aztec/end-to-end --production \ + && yarn cache clean + ELSE + ENV YARN_CACHE_FOLDER /yarn_cache + # TODO copy to other targets + # Use a mount for incremental builds locally. + # We cache NPM installs, typescript metadata and dest folders. + # TODO(AD): find safe way to cache noir contracts + RUN --mount type=cache,target=/node_cache \ + --mount type=cache,target=/yarn_cache \ + mkdir -p /node_cache/node_modules && mv /node_cache/node_modules . && \ + cp -r /node_cache/* . >/dev/null || echo "fresh build" && \ + ./bootstrap.sh && \ + yarn workspace @aztec/end-to-end run build:web && \ + yarn workspaces focus @aztec/end-to-end --production && \ + cp -r node_modules /node_cache/node_modules && \ + cp -r --parents */dest /node_cache && \ + find . -name '*.tsbuildinfo' -exec cp --parents {} /node_cache \; + END + SAVE ARTIFACT /build + + # Build web bundle for browser tests + RUN yarn workspace @aztec/end-to-end run build:web + RUN yarn workspaces focus @aztec/end-to-end --production && yarn cache clean + SAVE ARTIFACT /build + +end-to-end-minimal: + # end to end test runner + BUILD +aztec + FROM node:18.19.1-slim + RUN apt-get update && apt-get install jq chromium -y + ENV CHROME_BIN="/usr/bin/chromium" + COPY +end-to-end/build /build + WORKDIR /build/yarn-project/end-to-end + ENTRYPOINT ["yarn", "test"] + SAVE IMAGE aztecprotocol/cache:end-to-end diff --git a/yarn-project/accounts/.gitignore b/yarn-project/accounts/.gitignore deleted file mode 100644 index 7912fc48c7d..00000000000 --- a/yarn-project/accounts/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/src/artifacts diff --git a/yarn-project/aztec-faucet/.gitignore b/yarn-project/aztec-faucet/.gitignore deleted file mode 100644 index 81efe293f26..00000000000 --- a/yarn-project/aztec-faucet/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/data* diff --git a/yarn-project/aztec-node/.gitignore b/yarn-project/aztec-node/.gitignore deleted file mode 100644 index 81efe293f26..00000000000 --- a/yarn-project/aztec-node/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/data* diff --git a/yarn-project/aztec.js/.gitignore b/yarn-project/aztec.js/.gitignore deleted file mode 100644 index 03b565b6c24..00000000000 --- a/yarn-project/aztec.js/.gitignore +++ /dev/null @@ -1 +0,0 @@ -src/account_contract/artifacts \ No newline at end of file diff --git a/yarn-project/aztec/.gitignore b/yarn-project/aztec/.gitignore deleted file mode 100644 index 31dbbff57c2..00000000000 --- a/yarn-project/aztec/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/log diff --git a/yarn-project/circuits.js/.gitignore b/yarn-project/circuits.js/.gitignore deleted file mode 100644 index 6fc41e1039e..00000000000 --- a/yarn-project/circuits.js/.gitignore +++ /dev/null @@ -1 +0,0 @@ -fixtures/*.json diff --git a/yarn-project/docs/.gitignore b/yarn-project/docs/.gitignore deleted file mode 100644 index 53c37a16608..00000000000 --- a/yarn-project/docs/.gitignore +++ /dev/null @@ -1 +0,0 @@ -dist \ No newline at end of file diff --git a/yarn-project/end-to-end/.gitignore b/yarn-project/end-to-end/.gitignore deleted file mode 100644 index 48645e85aab..00000000000 --- a/yarn-project/end-to-end/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -addresses.json -/log -/data -/src/web/main.js -/src/web/main.js.LICENSE.txt \ No newline at end of file diff --git a/yarn-project/end-to-end/Earthfile b/yarn-project/end-to-end/Earthfile new file mode 100644 index 00000000000..c7228768a72 --- /dev/null +++ b/yarn-project/end-to-end/Earthfile @@ -0,0 +1,151 @@ + +VERSION 0.8 + +get-aztec: + FROM ubuntu:lunar + # for use_aztec_image to work, need to run earthly +build first or have image copied + ARG use_aztec_image=false + IF [ "$use_aztec_image" != false ] + FROM aztecprotocol/cache:aztec + ELSE + FROM ../+aztec + END + +get-end-to-end: + FROM ubuntu:lunar + # for use_end_to_end_image to work, need to run earthly +build first or have image copied + ARG use_end_to_end_image=false + IF [ $use_end_to_end_image != false ] + FROM aztecprotocol/cache:end-to-end + ELSE + FROM ../+end-to-end-minimal + END + +E2E_TEST: + FUNCTION + ARG test + ARG compose_file=./scripts/docker-compose.yml + LOCALLY + ENV TEST=$test + WITH DOCKER \ + --load aztecprotocol/aztec:latest=+get-aztec \ + --load aztecprotocol/end-to-end:latest=+get-end-to-end \ + --load ghcr.io/foundry-rs/foundry:nightly-de33b6af53005037b463318d2628b5cfcaf39916=../../foundry/+get + # Run our docker compose, ending whenever sandbox ends, filtering out noisy eth_getLogs + RUN docker compose -f $compose_file up --exit-code-from=sandbox --force-recreate + END + +# we could use a parameterized target, but these just print cleaner in earthly log + +e2e-block-building: + DO +E2E_TEST --test=e2e_block_building.test.ts + +e2e-nested-contract: + DO +E2E_TEST --test=e2e_nested_contract.test.ts + +e2e-static-calls: + DO +E2E_TEST --test=e2e_static_calls.test.ts + +e2e-delegate-calls: + DO +E2E_TEST --test=e2e_delegate_calls.test.ts + +e2e-non-contract-account: + DO +E2E_TEST --test=e2e_non_contract_account.test.ts + +e2e-cross-chain-messaging: + DO +E2E_TEST --test=e2e_cross_chain_messaging.test.ts + +e2e-crowdfunding-and-claim: + DO +E2E_TEST --test=e2e_crowdfunding_and_claim.test.ts + +e2e-public-cross-chain-messaging: + DO +E2E_TEST --test=e2e_public_cross_chain_messaging.test.ts + +e2e-public-to-private-messaging: + DO +E2E_TEST --test=e2e_public_to_private_messaging.test.ts + +e2e-account-contracts: + DO +E2E_TEST --test=e2e_account_contracts.test.ts + +e2e-escrow-contract: + DO +E2E_TEST --test=e2e_escrow_contract.test.ts + +e2e-inclusion-proofs-contract: + DO +E2E_TEST --test=e2e_inclusion_proofs_contract.test.ts + +e2e-pending-note-hashes-contract: + DO +E2E_TEST --test=e2e_pending_note_hashes_contract.test.ts + +e2e-ordering: + DO +E2E_TEST --test=e2e_ordering.test.ts + +uniswap-trade-on-l1-from-l2: + DO +E2E_TEST --test=uniswap_trade_on_l1_from_l2.test.ts + +integration-archiver-l1-to-l2: + DO +E2E_TEST --test=integration_archiver_l1_to_l2.test.ts + +integration-l1-publisher: + DO +E2E_TEST --test=integration_l1_publisher.test.ts + +e2e-cli: + DO +E2E_TEST --test=e2e_cli.test.ts + +e2e-persistence: + DO +E2E_TEST --test=e2e_persistence.test.ts --compose_file=./scripts/docker-compose-no-sandbox.yml + +e2e-browser: + DO +E2E_TEST --test=e2e_aztec_js_browser.test.ts + +e2e-card-game: + DO +E2E_TEST --test=e2e_card_game.test.ts + +e2e-avm-simulator: + ENV AVM_ENABLED=1 + DO +E2E_TEST --test=e2e_avm_simulator.test.ts + +e2e-fees: + DO +E2E_TEST --test=e2e_fees.test.ts + +e2e-dapp-subscription: + DO +E2E_TEST --test=e2e_dapp_subscription.test.ts + +pxe: + DO +E2E_TEST --test=pxe_sandbox.test.ts + +cli-docs-sandbox: + DO +E2E_TEST --test=cli_docs_sandbox.test.ts + +e2e-docs-examples: + ENV AVM_ENABLED=1 + DO +E2E_TEST --test=docs_examples_test.ts + +test-all: + BUILD +e2e-state-vars + BUILD +e2e-block-building + BUILD +e2e-nested-contract + BUILD +e2e-static-calls + BUILD +e2e-delegate-calls + BUILD +e2e-non-contract-account + BUILD +e2e-cross-chain-messaging + BUILD +e2e-crowdfunding-and-claim + BUILD +e2e-public-cross-chain-messaging + BUILD +e2e-public-to-private-messaging + BUILD +e2e-account-contracts + BUILD +e2e-escrow-contract + BUILD +e2e-inclusion-proofs-contract + BUILD +e2e-pending-note-hashes-contract + BUILD +e2e-ordering + BUILD +uniswap-trade-on-l1-from-l2 + BUILD +integration-archiver-l1-to-l2 + BUILD +integration-l1-publisher + BUILD +e2e-cli + BUILD +e2e-persistence + BUILD +e2e-browser + BUILD +e2e-card-game + BUILD +e2e-avm-simulator + BUILD +e2e-fees + BUILD +e2e-dapp-subscription + BUILD +pxe + BUILD +cli-docs-sandbox + BUILD +e2e-docs-examples diff --git a/yarn-project/end-to-end/scripts/docker-compose.yml b/yarn-project/end-to-end/scripts/docker-compose.yml index 0a73d344095..114fd5f9ad8 100644 --- a/yarn-project/end-to-end/scripts/docker-compose.yml +++ b/yarn-project/end-to-end/scripts/docker-compose.yml @@ -5,10 +5,12 @@ services: entrypoint: > sh -c ' if [ -n "$FORK_BLOCK_NUMBER" ] && [ -n "$FORK_URL" ]; then - exec anvil -p 8545 --host 0.0.0.0 --chain-id 31337 --fork-url "$FORK_URL" --fork-block-number "$FORK_BLOCK_NUMBER" + { anvil -p 8545 --host 0.0.0.0 --chain-id 31337 --fork-url "$FORK_URL" --fork-block-number "$FORK_BLOCK_NUMBER" ; echo $$? > .status ; } | grep -v eth_getLogs else - exec anvil -p 8545 --host 0.0.0.0 --chain-id 31337 - fi' + { anvil -p 8545 --host 0.0.0.0 --chain-id 31337 ; echo $$? > .status ; } | grep -v eth_getLogs + fi + # final status + grep -q '^0$$' .status' ports: - '8545:8545' @@ -40,6 +42,7 @@ services: PXE_URL: http://sandbox:8080 command: ${TEST:-./src/e2e_deploy_contract.test.ts} volumes: + # TODO(AD) currently earthly uses /build instead of /usr/src - ../log:/usr/src/yarn-project/end-to-end/log:rw depends_on: - sandbox diff --git a/yarn-project/entrypoints/.gitignore b/yarn-project/entrypoints/.gitignore deleted file mode 100644 index 7912fc48c7d..00000000000 --- a/yarn-project/entrypoints/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/src/artifacts diff --git a/yarn-project/l1-artifacts/.gitignore b/yarn-project/l1-artifacts/.gitignore deleted file mode 100644 index dc9b2375c7a..00000000000 --- a/yarn-project/l1-artifacts/.gitignore +++ /dev/null @@ -1 +0,0 @@ -generated \ No newline at end of file diff --git a/yarn-project/noir-compiler/.gitignore b/yarn-project/noir-compiler/.gitignore deleted file mode 100644 index 4e3795cbce5..00000000000 --- a/yarn-project/noir-compiler/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -target/ -proofs/ -Prover.toml -Verifier.toml \ No newline at end of file diff --git a/yarn-project/noir-contracts.js/.gitignore b/yarn-project/noir-contracts.js/.gitignore deleted file mode 100644 index afe0a2a36cb..00000000000 --- a/yarn-project/noir-contracts.js/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -target/ -/src -artifacts/ -codegenCache.json diff --git a/yarn-project/noir-protocol-circuits-types/.gitignore b/yarn-project/noir-protocol-circuits-types/.gitignore deleted file mode 100644 index bc5c9c9dc28..00000000000 --- a/yarn-project/noir-protocol-circuits-types/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -proofs/ -Prover.toml -Verifier.toml -src/target -src/crs -src/types diff --git a/yarn-project/package.json b/yarn-project/package.json index 5d83d4bbb73..5e5fbfb0a68 100644 --- a/yarn-project/package.json +++ b/yarn-project/package.json @@ -15,7 +15,7 @@ "build:fast": "yarn generate && tsc -b", "build:dev": "./watch.sh", "generate": "FORCE_COLOR=true yarn workspaces foreach --parallel --topological-dev --verbose run generate", - "clean": "yarn workspaces foreach -p -v run clean" + "clean": "yarn workspaces foreach -p -v run clean " }, "workspaces": [ "accounts", diff --git a/yarn-project/protocol-contracts/.gitignore b/yarn-project/protocol-contracts/.gitignore deleted file mode 100644 index 7912fc48c7d..00000000000 --- a/yarn-project/protocol-contracts/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/src/artifacts diff --git a/yarn-project/scripts/.gitignore b/yarn-project/scripts/.gitignore deleted file mode 100644 index ceeb05b4108..00000000000 --- a/yarn-project/scripts/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/tmp diff --git a/yarn-project/scripts/version_packages.sh b/yarn-project/scripts/version_packages.sh index bf631bd8ac1..d3865f324e7 100755 --- a/yarn-project/scripts/version_packages.sh +++ b/yarn-project/scripts/version_packages.sh @@ -4,5 +4,7 @@ set -eu if [ -n "$COMMIT_TAG" ]; then for workspace in $(yarn workspaces list --json | jq -r '.location'); do (cd $workspace && jq --arg v $COMMIT_TAG '.version = $v' package.json > _temp && mv _temp package.json) + # allow for versioning already-built packages + (cd $workspace && [ -f dest/package.json ] && jq --arg v $COMMIT_TAG '.version = $v' dest/package.json > _temp && mv _temp dest/package.json) done fi \ No newline at end of file