fix(ci): turn on earthly for everyone (#5423)

- Reenable and fix e2e-browser and e2e-docs-examples (which was broken in master, fixed by Palla) - Re-enstate single-threaded wasi for e2e-browser Note there is an intermittent issue with client_ivc_tests in ARM (more often than x86, anyway) so disabling for now until thats sorted - Reduce number of runners (2 x86, 1 ARM, 1 x86 smaller bench machine) because we now use e2e with github runners - Turn on earthly for everyone
AztecProtocol · Mar 28, 2024 · bea3fcb · bea3fcb
1 parent 8725c39
commit bea3fcb
Show file tree

Hide file tree

Showing 8 changed files with 122 additions and 129 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -32,7 +32,6 @@ jobs:
 
       - name: Test
         working-directory: ./yarn-project
-        if: github.actor == 'ludamad' || github.actor == 'charlielye' || github.actor == 'sklppy88' || github.actor == 'alexghr' || github.actor == 'benesjan' || github.actor == 'spalladino' || github.actor == 'just-mitch'
         run: |
           # push to registry
           earthly-cloud build x86 --push +build-end-to-end
@@ -68,10 +67,11 @@ jobs:
 
       - name: Test
         working-directory: ./yarn-project/end-to-end/
-        if: github.actor == 'ludamad' || github.actor == 'charlielye' || github.actor == 'sklppy88' || github.actor == 'alexghr' || github.actor == 'benesjan' || github.actor == 'spalladino' || github.actor == 'just-mitch'
         # We don't do much on arm, just run it on their builder
-        run: 
-          earthly-cloud build arm --no-output +${{ matrix.test }} --e2e_remote=true
+        run:
+          # Flags:
+          # - e2e_build=true to get images from a local build, not a dockerhub registry strategy
+          earthly-cloud build arm --no-output +${{ matrix.test }} --e2e_build=true
 
   # all the end-to-end integration tests for aztec
   e2e-x86:
@@ -98,18 +98,17 @@ jobs:
           - e2e-pending-note-hashes-contract
           - e2e-ordering
           - uniswap-trade-on-l1-from-l2
-          - integration-archiver-l1-to-l2
           - integration-l1-publisher
           - e2e-cli
           - e2e-persistence
-          # - e2e-browser # TODO failing
+          - e2e-browser
           - e2e-card-game
           - e2e-avm-simulator
           - e2e-fees
           - e2e-dapp-subscription
           - pxe
           - cli-docs-sandbox
-          # - e2e-docs-examples # TODO failing
+          - e2e-docs-examples
     # cancel if reran on same PR if exists, otherwise if on same commit
     concurrency:
       group: ${{ matrix.test }}-${{ github.event.pull_request.number || github.ref_name }}-x86
@@ -131,7 +130,6 @@ jobs:
 
       - name: Pull E2E Images
         working-directory: ./barretenberg/cpp/
-        if: github.actor == 'ludamad' || github.actor == 'charlielye' || github.actor == 'sklppy88' || github.actor == 'alexghr' || github.actor == 'benesjan' || github.actor == 'spalladino' || github.actor == 'just-mitch'
         run: |
           docker pull aztecprotocol/aztec-cache:$(git rev-parse HEAD)
           docker pull aztecprotocol/end-to-end-cache:$(git rev-parse HEAD)
@@ -140,7 +138,6 @@ jobs:
 
       - name: Test
         working-directory: ./yarn-project/end-to-end/
-        if: github.actor == 'ludamad' || github.actor == 'charlielye' || github.actor == 'sklppy88' || github.actor == 'alexghr' || github.actor == 'benesjan' || github.actor == 'spalladino' || github.actor == 'just-mitch'
         run: |
           earthly-cloud build x86 --no-output +${{ matrix.test }}
 
@@ -149,7 +146,8 @@ jobs:
     runs-on: ubuntu-latest
     env:
       EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }}
-    strategy: { matrix: { environment: [x86, arm] } }
+    strategy: { matrix: { environment: [x86] } }
+    # strategy: { matrix: { environment: [x86, arm] } }
     # cancel if reran on same PR if exists, otherwise if on same commit
     concurrency:
       group: bb-native-tests-${{ github.event.pull_request.number || github.ref_name }}-${{ matrix.environment }}
@@ -171,7 +169,6 @@ jobs:
 
       - name: Test
         working-directory: ./barretenberg/cpp/
-        if: github.actor == 'ludamad' || github.actor == 'charlielye' || github.actor == 'sklppy88' || github.actor == 'alexghr' || github.actor == 'benesjan' || github.actor == 'spalladino' || github.actor == 'just-mitch'
         run: |
           earthly-cloud build ${{ matrix.environment }} --no-output +test
 
@@ -205,10 +202,8 @@ jobs:
 
       - name: Client IVC Bench
         working-directory: ./barretenberg/cpp/
-        if: github.actor == 'ludamad' || github.actor == 'charlielye' || github.actor == 'sklppy88' || github.actor == 'alexghr' || github.actor == 'benesjan' || github.actor == 'spalladino' || github.actor == 'just-mitch'
         run: earthly-cloud bench x86 --no-output +bench-client-ivc
 
       - name: Ultrahonk Bench
         working-directory: ./barretenberg/cpp/
-        if: github.actor == 'ludamad' || github.actor == 'charlielye' || github.actor == 'sklppy88' || github.actor == 'alexghr' || github.actor == 'benesjan' || github.actor == 'spalladino' || github.actor == 'just-mitch'
         run: earthly-cloud bench x86 --no-output +bench-ultra-honk
diff --git a/barretenberg/cpp/Earthfile b/barretenberg/cpp/Earthfile
@@ -47,12 +47,21 @@ build-wasi-sdk:
     SAVE ARTIFACT wasi-sdk
     SAVE IMAGE --push aztecprotocol/cache:wasi-sdk-threads-v21.0-$TARGETARCH
 
-get-wasi-sdk:
+get-wasi-sdk-threads:
     ARG TARGETARCH
     # If this is failing, we need to run earthly --push +build-wasi-sdk
     FROM aztecprotocol/cache:wasi-sdk-threads-v21.0-$TARGETARCH
     SAVE ARTIFACT wasi-sdk
 
+get-wasi-sdk:
+    # NOTE: currently only works with x86
+    # TODO Align with above
+    FROM +source
+    COPY ./scripts/install-wasi-sdk.sh ./scripts/
+    RUN ./scripts/install-wasi-sdk.sh
+    # TODO better output name to mirror wasi-sdk
+    SAVE ARTIFACT src/wasi-sdk-20.0 wasi-sdk
+
 wasmtime:
     RUN curl https://wasmtime.dev/install.sh -sSf | bash
     SAVE ARTIFACT /root/.wasmtime/bin/wasmtime
@@ -81,10 +90,23 @@ preset-debug:
     SAVE ARTIFACT bin
 
 preset-wasm:
-    ARG BB_WASI_SDK=""
+    ARG TARGETARCH
+    FROM +source
+    IF [ $TARGETARCH = arm64 ]
+        # Just use threads for now
+        FROM +preset-wasm-threads
+        SAVE ARTIFACT bin
+    ELSE
+        COPY +get-wasi-sdk/wasi-sdk src/wasi-sdk
+        DO +RUN_CMAKE --configure="--preset wasm" --build="--target barretenberg.wasm"
+        RUN ../src/wasi-sdk/bin/llvm-strip ./bin/barretenberg.wasm
+        SAVE ARTIFACT bin
+        SAVE IMAGE --cache-hint
+    END
+
+preset-wasm-threads:
     FROM +source
-    # wasi-sdk
-    COPY +get-wasi-sdk/wasi-sdk src/wasi-sdk
+    COPY +get-wasi-sdk-threads/wasi-sdk src/wasi-sdk
     DO +RUN_CMAKE --configure="--preset wasm-threads" --build="--target barretenberg.wasm"
     RUN ../src/wasi-sdk/bin/llvm-strip ./bin/barretenberg.wasm
     SAVE ARTIFACT bin
@@ -132,7 +154,7 @@ bench-client-ivc:
 
 build: # default target
     BUILD +preset-release
-    BUILD +preset-wasm
+    BUILD +preset-wasm-threads
 
 test:
     BUILD +test-clang-format
@@ -190,7 +212,7 @@ BENCH_WASM:
     FUNCTION
     ARG target
     ARG args
-    FROM +preset-wasm
+    FROM +preset-wasm-threads
     DO +RUN_CMAKE --configure="--preset wasm-threads" --build="--target $target"
     COPY --dir ./srs_db/+build/. ../srs_db
     # install

diff --git a/barretenberg/cpp/scripts/install-wasi-sdk.sh b/barretenberg/cpp/scripts/install-wasi-sdk.sh
@@ -22,7 +22,6 @@ fi
 # Install the webassembly toolchain.
 mkdir -p src
 cd ./src
-# TODO(https://github.com/AztecProtocol/barretenberg/issues/865): is this needed?
 curl -s -L https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0-$OS.tar.gz | tar zxf -
 if [ "$OS" == "linux" ]; then
   # Exceptional linux case that provides an llvm build that works on Ubuntu 20.
@@ -35,5 +34,5 @@ else
   # Can remove once future releases are more stable.
   curl -s -L https://wasi-sdk.s3.eu-west-2.amazonaws.com/yamt-wasi-sysroot-20.0.threads.tgz | tar zxf -
 fi
-# TODO(https://github.com/AztecProtocol/barretenberg/issues/906): in the future this should use eartlhy and a 'SAVE ARTIFACT wasi-sdk AS LOCAL wasi-sdk'
+# TODO(https://github.com/AztecProtocol/barretenberg/issues/906): in the future this should use earthly and a 'SAVE ARTIFACT wasi-sdk AS LOCAL wasi-sdk'
 mv wasi-sdk-20.0+threads wasi-sdk
diff --git a/barretenberg/cpp/src/CMakeLists.txt b/barretenberg/cpp/src/CMakeLists.txt
@@ -23,7 +23,8 @@ if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
     endif()
     if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 18)
         # We target clang18 and need this, eventually warning should be fixed or this will be unconditional.
-        add_compile_options(-Wno-vla-cxx-extension)
+        add_compile_options(-Wno-vla-cxx-extension -pthread)
+        add_link_options(-Wl,--shared-memory)
     endif()
 endif()
 

diff --git a/barretenberg/ts/Earthfile b/barretenberg/ts/Earthfile
@@ -13,10 +13,9 @@ RUN yarn --immutable
 COPY --dir src *.json *.js *.cjs .
 
 # copy over wasm build from cpp folder
-COPY ../cpp/+preset-wasm/bin/barretenberg.wasm src/barretenberg_wasm/barretenberg-threads.wasm
-# TODO for now there is no real single-threaded WASM. See if anyone hits problems.
+COPY ../cpp/+preset-wasm-threads/bin/barretenberg.wasm src/barretenberg_wasm/barretenberg-threads.wasm
 COPY ../cpp/+preset-wasm/bin/barretenberg.wasm src/barretenberg_wasm/barretenberg.wasm
-COPY ../cpp/+preset-wasm/bin/barretenberg.wasm dest/node/barretenberg_wasm/barretenberg-threads.wasm
+COPY ../cpp/+preset-wasm-threads/bin/barretenberg.wasm dest/node/barretenberg_wasm/barretenberg-threads.wasm
 COPY ../cpp/+preset-wasm/bin/barretenberg.wasm dest/node-cjs/barretenberg_wasm/barretenberg-threads.wasm
 
 esm:

diff --git a/scripts/earthly-cloud b/scripts/earthly-cloud
@@ -28,12 +28,15 @@ fi
 # default sizes for build type
 if [ "$RUNNER_TYPE" == "build" ] ; then
   SIZE=4xlarge
-  NUMBER_OF_RUNNERS=4
+  NUMBER_OF_RUNNERS=2
+  if [ "$ARCH" = arm ] ; then
+    NUMBER_OF_RUNNERS=1
+  fi
   # TODO why cant we set this??
   # MAX_PARALLELISM=8
 elif [ "$RUNNER_TYPE" == "bench" ] ; then
   SIZE=2xlarge
-  NUMBER_OF_RUNNERS=2
+  NUMBER_OF_RUNNERS=1
   # MAX_PARALLELISM=1
 fi
 
@@ -55,15 +58,16 @@ RUNNER=$RUNNER_TYPE-$RUNNER_ID-$ARCH
 earthly sat --org aztec launch --size $SIZE --platform $PLATFORM $RUNNER || true
 # --remote-cache=aztecprotocol/cache:bb-native-tests 
 EARTHLY_FLAGS="-P --no-output --org aztec --sat $RUNNER"
+OUTPUT_FILE=$(mktemp)
 # capture output to handle earthly edge cases
 if $INTERACTIVE ; then
   # don't play nice with tee if interactive
   earthly $EARTHLY_FLAGS $@
-elif ! earthly $EARTHLY_FLAGS $@ 2>&1 | tee .output.txt >&2 ; then
+elif ! earthly $EARTHLY_FLAGS $@ 2>&1 | tee $OUTPUT_FILE >&2 ; then
   # we try earthly once, capturing output
   # if we get one of our (unfortunate) known failures, handle retries
   # TODO potentially handle other intermittent errors here
-  if grep 'failed to get edge: inconsistent graph state' .output.txt >/dev/null ; then
+  if grep 'failed to get edge: inconsistent graph state' $OUTPUT_FILE >/dev/null ; then
     # TODO when earthly is overloaded we sometimes get 
     # 'failed to solve: failed to get edge: inconsistent graph state'
     echo "Got 'inconsistent graph state'. Restarting earthly. See https://github.com/earthly/earthly/issues/2454'"

diff --git a/yarn-project/Earthfile b/yarn-project/Earthfile
@@ -29,18 +29,14 @@ build:
     # TODO: Replace puppeteer with puppeteer-core to avoid this.
     # TODO encapsulate in bash script for cleanliness
     ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
-    IF $EARTHLY_CI # Don't cache CI builds.
-        RUN ./bootstrap.sh && yarn workspaces focus @aztec/cli @aztec/aztec --production && yarn cache clean
-    ELSE
-        ENV YARN_CACHE_FOLDER /build/yarn-cache
-        # Use a mount for incremental builds locally.
-        RUN --mount type=cache,target=/build/node_modules_cache --mount type=cache,target=/build/yarn-cache \
-            mkdir -p node_modules_cache && \
-            mv node_modules_cache node_modules && \
-            ./bootstrap.sh && \
-            yarn workspaces focus @aztec/cli @aztec/aztec --production && \
-            cp -r node_modules node_modules_cache
-    END
+    ENV YARN_CACHE_FOLDER /build/yarn-cache
+    # Use a mount for incremental builds locally.
+    RUN --mount type=cache,target=/build/node_modules_cache --mount type=cache,target=/build/yarn-cache \
+        mkdir -p node_modules_cache && \
+        mv node_modules_cache node_modules && \
+        ./bootstrap.sh && \
+        yarn workspaces focus @aztec/cli @aztec/aztec --production && \
+        cp -r node_modules node_modules_cache
     SAVE ARTIFACT /build
 
 # TODO versioning flow at end before publish?
@@ -74,29 +70,10 @@ end-to-end:
     FROM +source
     # compilation artifacts for end-to-end
     # TODO encapsulate in bash script for cleanliness
-    ARG EARTHLY_CI
-    IF $EARTHLY_CI # Don't cache CI builds.
-        RUN ./bootstrap.sh && \
-            yarn workspace @aztec/end-to-end run build:web && \
-            yarn workspaces focus @aztec/end-to-end --production \
-            && yarn cache clean
-    ELSE
-        ENV YARN_CACHE_FOLDER /yarn_cache
-        # TODO copy to other targets
-        # Use a mount for incremental builds locally.
-        # We cache NPM installs, typescript metadata and dest folders.
-        # TODO(AD): find safe way to cache noir contracts
-        RUN   --mount type=cache,target=/node_cache \
-              --mount type=cache,target=/yarn_cache \
-            mkdir -p /node_cache/node_modules && mv /node_cache/node_modules . && \
-            cp -r /node_cache/* . >/dev/null || echo "fresh build" && \
-            ./bootstrap.sh && \
-            yarn workspace @aztec/end-to-end run build:web && \
-            yarn workspaces focus @aztec/end-to-end --production && \
-            cp -r node_modules /node_cache/node_modules && \
-            cp -r --parents */dest /node_cache && \
-            find . -name '*.tsbuildinfo' -exec cp --parents {} /node_cache \;
-    END
+    RUN ./bootstrap.sh && \
+        yarn workspace @aztec/end-to-end run build:web && \
+        yarn workspaces focus @aztec/end-to-end --production \
+        && yarn cache clean
     # Build web bundle for browser tests
     RUN yarn workspace @aztec/end-to-end run build:web
     RUN yarn workspaces focus @aztec/end-to-end --production && yarn cache clean