diff --git a/scripts/ci/assemble_e2e_benchmark.sh b/scripts/ci/assemble_e2e_benchmark.sh index 647e3cbcd97..90291ffa8b8 100755 --- a/scripts/ci/assemble_e2e_benchmark.sh +++ b/scripts/ci/assemble_e2e_benchmark.sh @@ -4,38 +4,37 @@ # and assembles a single file that shows the current state of the repository. [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace - -# Enter yarn-project dir. -cd "$(dirname $0)/../../yarn-project/" set -eu -PULL_REQUEST_ID=$1 -BRANCH=$2 BUCKET_NAME="aztec-ci-artifacts" -COMMIT_HASH=$3 +LOG_FOLDER="${LOG_FOLDER:-log}" +BENCH_FOLDER="${BENCH_FOLDER:-bench}" +COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" BASE_BENCH_PATH="" +BENCHMARK_FILE_JSON="${BENCH_FOLDER}/benchmark.json" +BASE_BENCHMARK_FILE_JSON="${BENCH_FOLDER}/base-benchmark.json" -# Paths from scripts/upload_logs_to_s3 -if [ "${BRANCH:-}" = "master" ]; then +# Paths from build-system/scripts/upload_logs_to_s3 +if [ "${CIRCLE_BRANCH:-}" = "master" ]; then LOG_SOURCE_FOLDER="logs-v1/master/$COMMIT_HASH" BARRETENBERG_BENCH_SOURCE_FOLDER="barretenberg-bench-v1/master/$COMMIT_HASH" BENCHMARK_TARGET_FILE="benchmarks-v1/master/$COMMIT_HASH.json" BENCHMARK_LATEST_FILE="benchmarks-v1/latest.json" -elif [ -n "$PULL_REQUEST_ID" ]; then - LOG_SOURCE_FOLDER="logs-v1/pulls/${PULL_REQUEST_ID##*/}" - BARRETENBERG_BENCH_SOURCE_FOLDER="barretenberg-bench-v1/pulls/${PULL_REQUEST_ID##*/}" - BENCHMARK_TARGET_FILE="benchmarks-v1/pulls/${PULL_REQUEST_ID##*/}.json" +elif [ -n "${CIRCLE_PULL_REQUEST:-}" ]; then + LOG_SOURCE_FOLDER="logs-v1/pulls/${CIRCLE_PULL_REQUEST##*/}" + BARRETENBERG_BENCH_SOURCE_FOLDER="barretenberg-bench-v1/pulls/${CIRCLE_PULL_REQUEST##*/}" + BENCHMARK_TARGET_FILE="benchmarks-v1/pulls/${CIRCLE_PULL_REQUEST##*/}.json" elif [ -n "${CIRCLE_TAG:-}" ]; then echo "Skipping benchmark run for ${CIRCLE_TAG} tagged release." exit 0 else - echo "Skipping benchmark run on branch ${BRANCH:-unknown}." + echo "Skipping benchmark run on branch ${CIRCLE_BRANCH:-unknown}." exit 0 fi -# Download benchmark log files from S3 LOG_SOURCE_FOLDER into local 'log' folder -mkdir -p log -aws s3 cp "s3://${BUCKET_NAME}/${LOG_SOURCE_FOLDER}/" log --exclude '*' --include 'bench*.jsonl' --recursive +# Download benchmark log files from S3 LOG_SOURCE_FOLDER into local LOG_FOLDER +mkdir -p $LOG_FOLDER +aws s3 cp "s3://${BUCKET_NAME}/${LOG_SOURCE_FOLDER}/" $LOG_FOLDER --exclude '*' --include 'bench*.jsonl' --recursive # Only generate the aggregated benchmark if we've managed to retrieve all the needed log files # If these runs were skipped due to no changes in their rebuild-patterns, then there's no need @@ -43,25 +42,60 @@ aws s3 cp "s3://${BUCKET_NAME}/${LOG_SOURCE_FOLDER}/" log --exclude '*' --includ # this skips the whole aggregation. For now, that's fine because all benchmark files have the # same rebuild pattern rules. But if that changes, then we'd need to go up in the commit history # to find the latest log files for the unchanged benchmarks. -EXPECTED_LOGS_COUNT=$(find end-to-end/src -type f -name "bench*.test.ts" | wc -l) -DOWNLOADED_LOGS_COUNT=$(find log -type f -name "*.jsonl" | wc -l) +EXPECTED_LOGS_COUNT=$(find yarn-project/end-to-end/src -type f -name "bench*.test.ts" | wc -l) +DOWNLOADED_LOGS_COUNT=$(find $LOG_FOLDER -type f -name "*.jsonl" | wc -l) if [ "$DOWNLOADED_LOGS_COUNT" -lt "$EXPECTED_LOGS_COUNT" ]; then echo Found $DOWNLOADED_LOGS_COUNT out of $EXPECTED_LOGS_COUNT benchmark log files in s3://${BUCKET_NAME}/${LOG_SOURCE_FOLDER}/. Exiting. exit 0 fi # Download barretenberg log files, these are direct benchmarks and separate from the above -aws s3 cp "s3://${BUCKET_NAME}/${BARRETENBERG_BENCH_SOURCE_FOLDER}/" log --exclude '*' --include '*_bench.json' --recursive +aws s3 cp "s3://${BUCKET_NAME}/${BARRETENBERG_BENCH_SOURCE_FOLDER}/" $LOG_FOLDER --exclude '*' --include '*_bench.json' --recursive # Generate the aggregated benchmark file -# TODO rename CIRCLE_PULL_REQUEST to PULL_REQUEST_ID -CIRCLE_PULL_REQUEST=$PULL_REQUEST_ID LOG_FOLDER=$(pwd)/log yarn workspace @aztec/scripts bench-aggregate -echo "generated: scripts/bench/benchmark.json" +mkdir -p $BENCH_FOLDER +CONTAINER_BENCH_FOLDER="/usr/src/yarn-project/bench" +CONTAINER_LOG_FOLDER="/usr/src/yarn-project/log" +export DOCKER_RUN_OPTS="\ + -v $(realpath $BENCH_FOLDER):${CONTAINER_BENCH_FOLDER}:rw \ + -e BENCH_FOLDER=${CONTAINER_BENCH_FOLDER} \ + -v $(realpath $LOG_FOLDER):${CONTAINER_LOG_FOLDER}:rw \ + -e LOG_FOLDER=${CONTAINER_LOG_FOLDER} \ + -e BASE_BENCH_PATH \ + -e AZTEC_BOT_COMMENTER_GITHUB_TOKEN \ + -e CIRCLE_PULL_REQUEST" +yarn-project/scripts/run_script.sh workspace @aztec/scripts bench-aggregate +echo "generated: $BENCHMARK_FILE_JSON" # Upload it to master or pulls -aws s3 cp scripts/bench/benchmark.json "s3://${BUCKET_NAME}/${BENCHMARK_TARGET_FILE}" +aws s3 cp $BENCHMARK_FILE_JSON "s3://${BUCKET_NAME}/${BENCHMARK_TARGET_FILE}" # If on master, also update the "latest" benchmark file if [ -n "${BENCHMARK_LATEST_FILE:-}" ]; then - aws s3 cp scripts/bench/benchmark.json "s3://${BUCKET_NAME}/${BENCHMARK_LATEST_FILE}" + aws s3 cp $BENCHMARK_FILE_JSON "s3://${BUCKET_NAME}/${BENCHMARK_LATEST_FILE}" +fi + +# If on a pull request, get the data from the most recent commit on master where it's available, +# generate a markdown comment, and post it on the pull request +if [ -n "${CIRCLE_PULL_REQUEST:-}" ]; then + MASTER_COMMIT_HASH=$(curl -s "https://api.github.com/repos/AztecProtocol/aztec-packages/pulls/${CIRCLE_PULL_REQUEST##*/}" | jq -r '.base.sha') + MASTER_COMMIT_HASHES=($(git log $MASTER_COMMIT_HASH --format="%H" -n 50)) + + set +e + echo "Searching for base benchmark data starting from commit $MASTER_COMMIT_HASH" + for commit_hash in "${MASTER_COMMIT_HASHES[@]}"; do + aws s3 cp "s3://${BUCKET_NAME}/benchmarks-v1/master/$commit_hash.json" $BASE_BENCHMARK_FILE_JSON + if [ $? -eq 0 ]; then + echo "Downloaded base data from commit $commit_hash" + export BASE_BENCH_PATH=master/$commit_hash + break; + fi + done + set -e + + if [ -z "${BASE_BENCH_PATH:-}" ]; then + echo "No base commit data found" + fi + + (yarn-project/scripts/run_script.sh workspace @aztec/scripts bench-comment && echo "commented on pr $CIRCLE_PULL_REQUEST") || echo "failed commenting on pr" fi