From 137c13e3dc33b98d2b641afcf30ca991e9f6071f Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Fri, 15 Mar 2024 10:26:53 -0300 Subject: [PATCH] chore: Use inotifywait to run generate in yarn-project (#5168) Replaces `yarn build:dev` at the root of yarn-packages with a script that leverages inotifywait to re-run `yarn generate` in the packages where it's needed. To achieve this, we define specialized `generate` commands based on the dependencies (l1 artifacts, noir contracts, or noir protocol circuits) and run the one depending on the modified folder. Since typescript watch doesn't seem to like if we modify too many files at a time, we kill it while we're doing codegen, and then restert it. Also, to make sure we don't run codegen before the compilation of the dependent artifacts is completed, we debounce the call by a few seconds. --- yarn-project/.gitignore | 2 + yarn-project/accounts/package.json | 3 +- yarn-project/accounts/package.local.json | 3 +- yarn-project/l1-artifacts/package.json | 3 +- yarn-project/noir-contracts.js/package.json | 3 +- .../noir-contracts.js/package.local.json | 3 +- .../noir-protocol-circuits-types/package.json | 3 +- yarn-project/package.json | 5 +- yarn-project/protocol-contracts/package.json | 3 +- .../protocol-contracts/package.local.json | 3 +- yarn-project/scripts/package.json | 10 ++- yarn-project/scripts/package.local.json | 9 +++ yarn-project/watch.sh | 78 +++++++++++++++++++ 13 files changed, 115 insertions(+), 13 deletions(-) create mode 100644 yarn-project/scripts/package.local.json create mode 100755 yarn-project/watch.sh diff --git a/yarn-project/.gitignore b/yarn-project/.gitignore index 697a1d7b922..a8ce42c3200 100644 --- a/yarn-project/.gitignore +++ b/yarn-project/.gitignore @@ -8,6 +8,8 @@ node_modules tsconfig.tsbuildinfo .eslintcache simulator/target +.debounce-* +.tsc.pid .yarn/* !.yarn/patches diff --git a/yarn-project/accounts/package.json b/yarn-project/accounts/package.json index ffc0e086f08..4888de8dbf2 100644 --- a/yarn-project/accounts/package.json +++ b/yarn-project/accounts/package.json @@ -24,7 +24,8 @@ }, "scripts": { "build": "yarn clean && yarn generate && tsc -b", - "generate": "./scripts/copy-contracts.sh", + "generate": "yarn generate:noir-contracts", + "generate:noir-contracts": "./scripts/copy-contracts.sh", "build:dev": "tsc -b --watch", "build:ts": "tsc -b", "clean": "rm -rf ./dest .tsbuildinfo ./src/artifacts", diff --git a/yarn-project/accounts/package.local.json b/yarn-project/accounts/package.local.json index ffe6a936683..c5987104cfc 100644 --- a/yarn-project/accounts/package.local.json +++ b/yarn-project/accounts/package.local.json @@ -1,7 +1,8 @@ { "scripts": { "build": "yarn clean && yarn generate && tsc -b", - "generate": "./scripts/copy-contracts.sh", + "generate": "yarn generate:noir-contracts", + "generate:noir-contracts": "./scripts/copy-contracts.sh", "build:dev": "tsc -b --watch", "build:ts": "tsc -b", "clean": "rm -rf ./dest .tsbuildinfo ./src/artifacts" diff --git a/yarn-project/l1-artifacts/package.json b/yarn-project/l1-artifacts/package.json index 57e271c6927..5bf298bbb3c 100644 --- a/yarn-project/l1-artifacts/package.json +++ b/yarn-project/l1-artifacts/package.json @@ -15,7 +15,8 @@ "clean": "rm -rf ./dest ./generated .tsbuildinfo", "formatting": "run -T prettier --check ./generated && run -T eslint ./generated", "formatting:fix": "run -T prettier -w ./generated", - "generate": "bash scripts/generate-artifacts.sh" + "generate": "yarn generate:l1-contracts", + "generate:l1-contracts": "bash scripts/generate-artifacts.sh" }, "dependencies": { "tslib": "^2.4.0" diff --git a/yarn-project/noir-contracts.js/package.json b/yarn-project/noir-contracts.js/package.json index 6227dede094..b8c6324b1c3 100644 --- a/yarn-project/noir-contracts.js/package.json +++ b/yarn-project/noir-contracts.js/package.json @@ -14,7 +14,8 @@ "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --passWithNoTests", - "generate": "./scripts/generate-types.sh && run -T prettier -w ./src" + "generate": "yarn generate:noir-contracts", + "generate:noir-contracts": "./scripts/generate-types.sh && run -T prettier -w ./src --loglevel warn" }, "inherits": [ "../package.common.json", diff --git a/yarn-project/noir-contracts.js/package.local.json b/yarn-project/noir-contracts.js/package.local.json index a61ddc84d33..5dd3e0100f9 100644 --- a/yarn-project/noir-contracts.js/package.local.json +++ b/yarn-project/noir-contracts.js/package.local.json @@ -1,7 +1,8 @@ { "scripts": { "build": "yarn clean && yarn generate", - "generate": "./scripts/generate-types.sh && run -T prettier -w ./src", + "generate": "yarn generate:noir-contracts", + "generate:noir-contracts": "./scripts/generate-types.sh && run -T prettier -w ./src --loglevel warn", "clean": "rm -rf .tsbuildinfo ./artifacts ./codegenCache.json" } } diff --git a/yarn-project/noir-protocol-circuits-types/package.json b/yarn-project/noir-protocol-circuits-types/package.json index 8ca06e76d66..0656a50551b 100644 --- a/yarn-project/noir-protocol-circuits-types/package.json +++ b/yarn-project/noir-protocol-circuits-types/package.json @@ -12,7 +12,8 @@ "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "NODE_OPTIONS='--max-old-space-size=8096' run -T eslint --fix ./src && run -T prettier -w ./src", "formatting:fix:types": "NODE_OPTIONS='--max-old-space-size=8096' run -T eslint --fix ./src/types && run -T prettier -w ./src/types", - "generate": "mkdir -p ./src/target && cp ../../noir-projects/noir-protocol-circuits/target/* ./src/target && node --no-warnings --loader ts-node/esm src/scripts/generate_ts_from_abi.ts && run -T prettier -w ./src/types", + "generate": "yarn generate:noir-circuits", + "generate:noir-circuits": "mkdir -p ./src/target && cp ../../noir-projects/noir-protocol-circuits/target/* ./src/target && node --no-warnings --loader ts-node/esm src/scripts/generate_ts_from_abi.ts && run -T prettier -w ./src/types", "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --passWithNoTests" }, "jest": { diff --git a/yarn-project/package.json b/yarn-project/package.json index c2cb5c60bc3..5d83d4bbb73 100644 --- a/yarn-project/package.json +++ b/yarn-project/package.json @@ -13,7 +13,7 @@ "test": "FORCE_COLOR=true yarn workspaces foreach --exclude @aztec/aztec3-packages --exclude @aztec/end-to-end -p -v run test", "build": "FORCE_COLOR=true yarn workspaces foreach --parallel --topological-dev --verbose --exclude @aztec/aztec3-packages --exclude @aztec/docs run build", "build:fast": "yarn generate && tsc -b", - "build:dev": "tsc -b tsconfig.json --watch", + "build:dev": "./watch.sh", "generate": "FORCE_COLOR=true yarn workspaces foreach --parallel --topological-dev --verbose run generate", "clean": "yarn workspaces foreach -p -v run clean" }, @@ -49,8 +49,7 @@ "sequencer-client", "scripts", "types", - "world-state", - "yarn-project-base" + "world-state" ], "prettier": "@aztec/foundation/prettier", "devDependencies": { diff --git a/yarn-project/protocol-contracts/package.json b/yarn-project/protocol-contracts/package.json index 4c370ceb3ab..7bd6ed8c27d 100644 --- a/yarn-project/protocol-contracts/package.json +++ b/yarn-project/protocol-contracts/package.json @@ -19,7 +19,8 @@ }, "scripts": { "build": "yarn clean && yarn generate && tsc -b", - "generate": "./scripts/copy-contracts.sh", + "generate": "yarn generate:noir-contracts", + "generate:noir-contracts": "./scripts/copy-contracts.sh", "build:dev": "tsc -b --watch", "build:ts": "tsc -b", "clean": "rm -rf ./dest .tsbuildinfo ./src/artifacts", diff --git a/yarn-project/protocol-contracts/package.local.json b/yarn-project/protocol-contracts/package.local.json index ffe6a936683..c5987104cfc 100644 --- a/yarn-project/protocol-contracts/package.local.json +++ b/yarn-project/protocol-contracts/package.local.json @@ -1,7 +1,8 @@ { "scripts": { "build": "yarn clean && yarn generate && tsc -b", - "generate": "./scripts/copy-contracts.sh", + "generate": "yarn generate:noir-contracts", + "generate:noir-contracts": "./scripts/copy-contracts.sh", "build:dev": "tsc -b --watch", "build:ts": "tsc -b", "clean": "rm -rf ./dest .tsbuildinfo ./src/artifacts" diff --git a/yarn-project/scripts/package.json b/yarn-project/scripts/package.json index 2025fbb4d9a..154b80198a5 100644 --- a/yarn-project/scripts/package.json +++ b/yarn-project/scripts/package.json @@ -18,10 +18,16 @@ "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", "start:dev": "tsc-watch -p tsconfig.json --onSuccess 'yarn start'", "start": "node ./dest/index.js", - "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --passWithNoTests" + "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --passWithNoTests", + "generate:noir-circuits": "echo Noop", + "generate:noir-contracts": "echo Noop", + "generate:l1-contracts": "echo Noop", + "generate:why-these-noops": "echo These noops are here because `yarn workspaces foreach` runs the specified command in the packages that contain it only if two or more packages define it, otherwise it's run everywhere. So we just define these noops as commands to ensure they behave as they should when running watch.sh.", + "generate:why-these-comments": "echo JSON does not support comments, so we just define these commands for documentation sake." }, "inherits": [ - "../package.common.json" + "../package.common.json", + "./package.local.json" ], "dependencies": { "@aztec/circuit-types": "workspace:^", diff --git a/yarn-project/scripts/package.local.json b/yarn-project/scripts/package.local.json new file mode 100644 index 00000000000..05cdf4290ec --- /dev/null +++ b/yarn-project/scripts/package.local.json @@ -0,0 +1,9 @@ +{ + "scripts": { + "generate:noir-circuits": "echo Noop", + "generate:noir-contracts": "echo Noop", + "generate:l1-contracts": "echo Noop", + "generate:why-these-noops": "echo These noops are here because `yarn workspaces foreach` runs the specified command in the packages that contain it only if two or more packages define it, otherwise it's run everywhere. So we just define these noops as commands to ensure they behave as they should when running watch.sh.", + "generate:why-these-comments": "echo JSON does not support comments, so we just define these commands for documentation sake." + } +} \ No newline at end of file diff --git a/yarn-project/watch.sh b/yarn-project/watch.sh new file mode 100755 index 00000000000..250de4e6d7d --- /dev/null +++ b/yarn-project/watch.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +set -eu + +DEBOUNCE_DURATION=3 # Set a high duration for debounce since nargo build may pause for a long time during a compilation +INOTIFY_EVENTS="modify,create,delete,move" +NOIR_CONTRACTS_OUT_DIR="../noir-projects/noir-contracts/target/" +NOIR_CIRCUITS_OUT_DIR="../noir-projects/noir-protocol-circuits/target/" +L1_CONTRACTS_OUT_DIR="../l1-contracts/out/" + +# Debounce any command sent here. Grouped by command name and first arg. +debounce() { + local group_id="$1-$2" + local run_id=$(uuidgen) + echo "$run_id" > ".debounce-$group_id" + ( + sleep $DEBOUNCE_DURATION; + local current_id=$(cat ".debounce-$group_id"); + if [ "$run_id" = "${current_id}" ]; then + "$@" + fi + ) & +} + +# Start typescript watch process in the background and store process ID in a file +start_tsc_watch() { + yarn tsc -b tsconfig.json --watch & + TSC_PID=$! + echo "$TSC_PID" > .tsc.pid +} + +# Stops the typescript watch process +stop_tsc_watch() { + local tsc_pid=$(cat ".tsc.pid"); + kill $tsc_pid || true +} + +# Kill typescript, run a yarn generate, and restart typescript +run_generate() { + echo "Change detected at $1" + stop_tsc_watch + FORCE_COLOR=true yarn workspaces foreach --parallel --topological-dev --verbose run generate:$1 + echo "Generate complete, restarting typescript..." + sleep 3 + start_tsc_watch +} + +# Remove all temp files with process or run ids on exit +cleanup() { + rm .tsc.pid || true + rm .debounce-* || true +} +trap cleanup EXIT + +# Start tsc watch in background +start_tsc_watch + +# Watch for changes in the output directories +while true; do + folder=$(inotifywait --format '%w' --quiet --recursive --event $INOTIFY_EVENTS $NOIR_CONTRACTS_OUT_DIR $NOIR_CIRCUITS_OUT_DIR $L1_CONTRACTS_OUT_DIR) + case $folder in + "$NOIR_CONTRACTS_OUT_DIR") + debounce run_generate "noir-contracts" + ;; + "$NOIR_CIRCUITS_OUT_DIR") + debounce run_generate "noir-circuits" + ;; + "$L1_CONTRACTS_OUT_DIR"*) + debounce run_generate "l1-contracts" + ;; + *) + echo "Change at $folder not matched with any project" + exit 1 + ;; + esac +done + + +