Skip to content

Commit

Permalink
fix: make earthly more parallel (#5747)
Browse files Browse the repository at this point in the history
If you have FROM +dependency, its a good idea to prefetch above it with
BUILD +later-target commands
  • Loading branch information
ludamad committed Apr 15, 2024
1 parent 66dc509 commit 9734455
Show file tree
Hide file tree
Showing 4 changed files with 17 additions and 5 deletions.
5 changes: 4 additions & 1 deletion barretenberg/cpp/Earthfile
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,7 @@ bench-binaries:

# Runs on the bench image, sent from the builder runner
bench-ultra-honk:
BUILD +wasmtime # prefetch
FROM +source
COPY --dir +bench-binaries/* .
# install SRS needed for proving
Expand All @@ -198,6 +199,7 @@ bench-ultra-honk:
RUN cd wasm && wasmtime run --env HARDWARE_CONCURRENCY=16 -Wthreads=y -Sthreads=y --dir=".." ./bin/ultra_honk_bench --benchmark_filter="construct_proof_ultrahonk_power_of_2/20$"

bench-client-ivc:
BUILD +wasmtime # prefetch
FROM +source
COPY --dir +bench-binaries/* .
# install SRS needed for proving
Expand Down Expand Up @@ -232,9 +234,10 @@ test-clang-format:
RUN ./format.sh check

test:
BUILD +test-clang-format
BUILD ./srs_db/+build # prefetch
FROM +source
COPY --dir +test-binaries/build build
BUILD +test-clang-format
FROM +preset-release-assert-test
COPY --dir ./srs_db/+build/. srs_db
RUN cd build && GTEST_COLOR=1 ctest -j$(nproc) --output-on-failure
2 changes: 1 addition & 1 deletion boxes/Earthfile
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,4 @@ build:
ENV AZTEC_CLI=/usr/src/yarn-project/cli/aztec-cli-dest
RUN yarn && yarn build
RUN npx -y playwright@1.42 install --with-deps
ENTRYPOINT ["/bin/sh", "-c"]
ENTRYPOINT ["/bin/sh", "-c"]
7 changes: 4 additions & 3 deletions noir/Earthfile
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,14 @@ nargo:
SAVE IMAGE aztecprotocol/nargo

packages:
BUILD ../barretenberg/ts/+build # prefetch
FROM node:20

RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
RUN echo 'source $HOME/.cargo/env' >> $HOME/.bashrc
RUN apt update && apt install -y jq libc++1

# `noir-repo` is nested inside of `noir` so we copy `bb.js` as such to account
# `noir-repo` is nested inside of `noir` so we copy `bb.js` as such to account
# for the extra nested folder specified in portalled package paths
COPY ../barretenberg/ts/+build/build /usr/src/../barretenberg/ts

Expand Down Expand Up @@ -105,8 +106,8 @@ build:
# FROM scratch
# COPY --from=builder /usr/src/noir/README.md /usr/src/noir/README.md

# TODO
# test:
# TODO
# test:
# FROM rust:bullseye
# ARG COMMIT_HASH
# ENV COMMIT_HASH=${COMMIT_HASH}
Expand Down
8 changes: 8 additions & 0 deletions yarn-project/Earthfile
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,11 @@ deps:
RUN ln -s /usr/src/yarn-project/node_modules /usr/src/node_modules

build:
# Prefetch targets to not wait for +deps.
BUILD ../barretenberg/cpp/+preset-release
BUILD ../noir/+nargo
BUILD ../noir-projects/+build
BUILD ../l1-contracts/+build
FROM +deps
RUN apt update && apt install -y jq curl perl && rm -rf /var/lib/apt/lists/* && apt-get clean

Expand Down Expand Up @@ -82,6 +87,9 @@ all:

# for use with yarn-project/end-to-end and its e2e_mode=cache option
export-end-to-end:
# Prefetch targets to build in parallel.
BUILD +end-to-end
BUILD +aztec
ARG EARTHLY_GIT_HASH
# pushes the foundry image to local docker images
FROM ../foundry/+build
Expand Down

0 comments on commit 9734455

Please sign in to comment.