From 37134e6c35ef26659c0ed35c0221ccf3558e3ae2 Mon Sep 17 00:00:00 2001 From: spypsy Date: Tue, 27 Feb 2024 12:17:36 +0000 Subject: [PATCH 1/8] parallelize noir-contracts.js codegen --- .../scripts/generate-types.sh | 29 ++++++++----------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/yarn-project/noir-contracts.js/scripts/generate-types.sh b/yarn-project/noir-contracts.js/scripts/generate-types.sh index 8532ef04ca3..7143455777e 100755 --- a/yarn-project/noir-contracts.js/scripts/generate-types.sh +++ b/yarn-project/noir-contracts.js/scripts/generate-types.sh @@ -4,37 +4,32 @@ set -euo pipefail OUT_DIR="./src" INDEX="$OUT_DIR/index.ts" -rm -rf $OUT_DIR && mkdir -p $OUT_DIR +rm -rf "$OUT_DIR" && mkdir -p "$OUT_DIR" -# +# Check for .json files existence if ! ls ../../noir-projects/noir-contracts/target/*.json >/dev/null 2>&1; then echo "Error: No .json files found in noir-contracts/target folder." echo "Make sure noir-contracts is built before running this script." exit 1 fi -# Generate index.ts header. -echo "// Auto generated module - do not edit!" >$INDEX +# Generate index.ts header +echo "// Auto generated module - do not edit!" >"$INDEX" # Ensure the artifacts directory exists mkdir -p artifacts -for ABI in $(find ../../noir-projects/noir-contracts/target -maxdepth 1 -type f ! -name 'debug_*' -name '*.json'); do - # Extract the filename from the path - filename=$(basename "$ABI") - - # Copy the JSON file to the artifacts folder - cp "$ABI" "artifacts/$filename" - - # Generate the contract name for referencing in the codegen command and index +# Parallel codegen +export OUT_DIR +export INDEX +find ../../noir-projects/noir-contracts/target -maxdepth 1 -type f ! -name 'debug_*' -name '*.json' | parallel ' + filename=$(basename {}) + cp {} "artifacts/$filename" CONTRACT=$(jq -r .name "artifacts/$filename") - echo "Creating types for $CONTRACT using artifacts/$filename..." node --no-warnings ../noir-compiler/dest/cli.js codegen -o $OUT_DIR --ts "artifacts/$filename" - - # Add contract import/export to index.ts. - echo "export * from './${CONTRACT}.js';" >>$INDEX -done + echo "export * from '\''./${CONTRACT}.js'\'';" >> $INDEX +' echo "Formatting..." yarn formatting:fix From aa1d6c5cb3aed145b058c6be027e4a9604327ac0 Mon Sep 17 00:00:00 2001 From: spypsy Date: Tue, 27 Feb 2024 13:21:00 +0000 Subject: [PATCH 2/8] add parallel --- yarn-project/yarn-project-base/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/yarn-project-base/Dockerfile b/yarn-project/yarn-project-base/Dockerfile index b53ed652bc0..85c7eea2164 100644 --- a/yarn-project/yarn-project-base/Dockerfile +++ b/yarn-project/yarn-project-base/Dockerfile @@ -46,7 +46,7 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/bb.js as bb.js FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-packages as noir-packages FROM node:18.19.0 -RUN apt update && apt install -y jq curl perl && rm -rf /var/lib/apt/lists/* && apt-get clean +RUN apt update && apt install -y jq curl perl parallel && rm -rf /var/lib/apt/lists/* && apt-get clean # Copy in portalled packages. COPY --from=bb.js /usr/src/barretenberg/ts /usr/src/barretenberg/ts From d101e538eaa7d8e936728337d77fb0bb9d60464d Mon Sep 17 00:00:00 2001 From: spypsy Date: Tue, 27 Feb 2024 14:06:18 +0000 Subject: [PATCH 3/8] improve codegen.ts --- yarn-project/noir-compiler/src/cli/codegen.ts | 49 ++++++++++++++++++- yarn-project/noir-contracts.js/.gitignore | 1 + yarn-project/noir-contracts.js/package.json | 2 +- .../scripts/generate-types.sh | 23 ++++----- 4 files changed, 59 insertions(+), 16 deletions(-) diff --git a/yarn-project/noir-compiler/src/cli/codegen.ts b/yarn-project/noir-compiler/src/cli/codegen.ts index 19c90f0a551..1f2656e8ebb 100644 --- a/yarn-project/noir-compiler/src/cli/codegen.ts +++ b/yarn-project/noir-compiler/src/cli/codegen.ts @@ -1,11 +1,16 @@ +/* eslint-disable no-console */ import { loadContractArtifact } from '@aztec/types/abi'; -import { mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from 'fs'; +import crypto from 'crypto'; +import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from 'fs'; import path from 'path'; import { generateNoirContractInterface } from '../contract-interface-gen/noir.js'; import { generateTypescriptContractInterface } from '../contract-interface-gen/typescript.js'; +const cacheFilePath = './codegenCache.json'; +let cache: Record = {}; + /** Generate code options */ type GenerateCodeOptions = { /** Typescript */ ts?: boolean; /** Noir */ nr?: boolean }; @@ -13,10 +18,13 @@ type GenerateCodeOptions = { /** Typescript */ ts?: boolean; /** Noir */ nr?: bo * Generates Noir interface or Typescript interface for a folder or single file from a Noir compilation artifact. */ export function generateCode(outputPath: string, fileOrDirPath: string, opts: GenerateCodeOptions = {}) { + readCache(); const stats = statSync(fileOrDirPath); if (stats.isDirectory()) { - const files = readdirSync(fileOrDirPath).filter(file => file.endsWith('.json') && !file.startsWith('debug_')); + const files = readdirSync(fileOrDirPath, { recursive: true, encoding: 'utf-8' }).filter( + file => file.endsWith('.json') && !file.startsWith('debug_'), + ); for (const file of files) { const fullPath = path.join(fileOrDirPath, file); generateFromNoirAbi(outputPath, fullPath, opts); @@ -24,12 +32,21 @@ export function generateCode(outputPath: string, fileOrDirPath: string, opts: Ge } else if (stats.isFile()) { generateFromNoirAbi(outputPath, fileOrDirPath, opts); } + writeCache(); } /** * Generates Noir interface or Typescript interface for a single file Noir compilation artifact. */ function generateFromNoirAbi(outputPath: string, noirAbiPath: string, opts: GenerateCodeOptions = {}) { + const contractName = path.basename(noirAbiPath); + const currentHash = generateFileHash(noirAbiPath); + + if (isCacheValid(contractName, currentHash)) { + console.log(`${contractName} has not changed. Skipping generation.`); + return; + } + const contract = JSON.parse(readFileSync(noirAbiPath, 'utf8')); const aztecAbi = loadContractArtifact(contract); const { nr, ts } = opts; @@ -52,4 +69,32 @@ function generateFromNoirAbi(outputPath: string, noirAbiPath: string, opts: Gene const tsWrapper = generateTypescriptContractInterface(aztecAbi, relativeArtifactPath); writeFileSync(`${outputPath}/${aztecAbi.name}.ts`, tsWrapper); } + updateCache(contractName, currentHash); +} + +function generateFileHash(filePath: string) { + const fileBuffer = readFileSync(filePath); + const hashSum = crypto.createHash('sha256'); + hashSum.update(fileBuffer); + const hex = hashSum.digest('hex'); + return hex; +} + +function readCache(): void { + if (existsSync(cacheFilePath)) { + const cacheRaw = readFileSync(cacheFilePath, 'utf8'); + cache = JSON.parse(cacheRaw); + } +} + +function writeCache(): void { + writeFileSync(cacheFilePath, JSON.stringify(cache, null, 2), 'utf8'); +} + +function isCacheValid(contractName: string, currentHash: string): boolean { + return cache[contractName] === currentHash; +} + +function updateCache(contractName: string, hash: string): void { + cache[contractName] = hash; } diff --git a/yarn-project/noir-contracts.js/.gitignore b/yarn-project/noir-contracts.js/.gitignore index 1f61ab7482c..afe0a2a36cb 100644 --- a/yarn-project/noir-contracts.js/.gitignore +++ b/yarn-project/noir-contracts.js/.gitignore @@ -1,3 +1,4 @@ target/ /src artifacts/ +codegenCache.json diff --git a/yarn-project/noir-contracts.js/package.json b/yarn-project/noir-contracts.js/package.json index 10195fd680b..3b279fd7498 100644 --- a/yarn-project/noir-contracts.js/package.json +++ b/yarn-project/noir-contracts.js/package.json @@ -10,7 +10,7 @@ "scripts": { "build": "yarn clean && yarn build:contracts", "build:dev": "tsc -b --watch", - "clean": "rm -rf .tsbuildinfo ./artifacts", + "clean": "rm -rf .tsbuildinfo ./artifacts ./scripts/codegenCache.json ./src", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --passWithNoTests", diff --git a/yarn-project/noir-contracts.js/scripts/generate-types.sh b/yarn-project/noir-contracts.js/scripts/generate-types.sh index 7143455777e..5f3f3228e5f 100755 --- a/yarn-project/noir-contracts.js/scripts/generate-types.sh +++ b/yarn-project/noir-contracts.js/scripts/generate-types.sh @@ -4,8 +4,6 @@ set -euo pipefail OUT_DIR="./src" INDEX="$OUT_DIR/index.ts" -rm -rf "$OUT_DIR" && mkdir -p "$OUT_DIR" - # Check for .json files existence if ! ls ../../noir-projects/noir-contracts/target/*.json >/dev/null 2>&1; then echo "Error: No .json files found in noir-contracts/target folder." @@ -19,17 +17,16 @@ echo "// Auto generated module - do not edit!" >"$INDEX" # Ensure the artifacts directory exists mkdir -p artifacts -# Parallel codegen -export OUT_DIR -export INDEX -find ../../noir-projects/noir-contracts/target -maxdepth 1 -type f ! -name 'debug_*' -name '*.json' | parallel ' - filename=$(basename {}) - cp {} "artifacts/$filename" - CONTRACT=$(jq -r .name "artifacts/$filename") - echo "Creating types for $CONTRACT using artifacts/$filename..." - node --no-warnings ../noir-compiler/dest/cli.js codegen -o $OUT_DIR --ts "artifacts/$filename" - echo "export * from '\''./${CONTRACT}.js'\'';" >> $INDEX -' +for ABI in $(find ../../noir-projects/noir-contracts/target -maxdepth 1 -type f ! -name 'debug_*' -name '*.json'); do + # Extract the filename from the path + filename=$(basename "$ABI") + + # Copy the JSON file to the artifacts folder + cp "$ABI" "artifacts/$filename" +done + +# Generate types for the contracts +node --no-warnings ../noir-compiler/dest/cli.js codegen -o $OUT_DIR --ts artifacts echo "Formatting..." yarn formatting:fix From c2f1259dbbee5e9ec5ab6e2095bffa4420153cf4 Mon Sep 17 00:00:00 2001 From: spypsy Date: Tue, 27 Feb 2024 14:27:59 +0000 Subject: [PATCH 4/8] fixes --- yarn-project/bootstrap.sh | 2 +- yarn-project/noir-contracts.js/package.json | 2 +- yarn-project/noir-contracts.js/scripts/generate-types.sh | 8 ++++++++ yarn-project/yarn-project-base/Dockerfile | 2 +- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/yarn-project/bootstrap.sh b/yarn-project/bootstrap.sh index cbbf1fee8ae..78480fa5709 100755 --- a/yarn-project/bootstrap.sh +++ b/yarn-project/bootstrap.sh @@ -41,7 +41,7 @@ echo "Building noir compiler..." yarn workspace @aztec/noir-compiler build # Builds noir contracts (TODO: move this stage pre yarn-project). Generates typescript wrappers. echo "Building contracts from noir-contracts..." -yarn workspace @aztec/noir-contracts.js build:contracts +yarn workspace @aztec/noir-contracts.js build # Bundle compiled contracts into other packages echo "Copying account contracts..." yarn workspace @aztec/accounts build:copy-contracts diff --git a/yarn-project/noir-contracts.js/package.json b/yarn-project/noir-contracts.js/package.json index 3b279fd7498..b54febafcbb 100644 --- a/yarn-project/noir-contracts.js/package.json +++ b/yarn-project/noir-contracts.js/package.json @@ -10,7 +10,7 @@ "scripts": { "build": "yarn clean && yarn build:contracts", "build:dev": "tsc -b --watch", - "clean": "rm -rf .tsbuildinfo ./artifacts ./scripts/codegenCache.json ./src", + "clean": "rm -rf .tsbuildinfo ./artifacts ./codegenCache.json", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --passWithNoTests", diff --git a/yarn-project/noir-contracts.js/scripts/generate-types.sh b/yarn-project/noir-contracts.js/scripts/generate-types.sh index 5f3f3228e5f..de07e0637ee 100755 --- a/yarn-project/noir-contracts.js/scripts/generate-types.sh +++ b/yarn-project/noir-contracts.js/scripts/generate-types.sh @@ -4,6 +4,8 @@ set -euo pipefail OUT_DIR="./src" INDEX="$OUT_DIR/index.ts" +mkdir -p $OUT_DIR + # Check for .json files existence if ! ls ../../noir-projects/noir-contracts/target/*.json >/dev/null 2>&1; then echo "Error: No .json files found in noir-contracts/target folder." @@ -28,5 +30,11 @@ done # Generate types for the contracts node --no-warnings ../noir-compiler/dest/cli.js codegen -o $OUT_DIR --ts artifacts +# Append exports for each generated TypeScript file to index.ts +find "$OUT_DIR" -maxdepth 1 -type f -name '*.ts' ! -name 'index.ts' | while read -r TS_FILE; do + CONTRACT_NAME=$(basename "$TS_FILE" .ts) # Remove the .ts extension to get the contract name + echo "export * from './${CONTRACT_NAME}.js';" >>"$INDEX" +done + echo "Formatting..." yarn formatting:fix diff --git a/yarn-project/yarn-project-base/Dockerfile b/yarn-project/yarn-project-base/Dockerfile index 44dcf806deb..62c4b55fb42 100644 --- a/yarn-project/yarn-project-base/Dockerfile +++ b/yarn-project/yarn-project-base/Dockerfile @@ -46,7 +46,7 @@ FROM aztecprotocol/bb.js as bb.js FROM aztecprotocol/noir-packages as noir-packages FROM node:18.19.0 -RUN apt update && apt install -y jq curl perl parallel && rm -rf /var/lib/apt/lists/* && apt-get clean +RUN apt update && apt install -y jq curl perl && rm -rf /var/lib/apt/lists/* && apt-get clean # Copy in portalled packages. COPY --from=bb.js /usr/src/barretenberg/ts /usr/src/barretenberg/ts From 0e5354183d12df78d707e8f9636cd7451689f960 Mon Sep 17 00:00:00 2001 From: spypsy Date: Tue, 27 Feb 2024 18:15:31 +0000 Subject: [PATCH 5/8] update package.local.json --- yarn-project/noir-contracts.js/package.local.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/noir-contracts.js/package.local.json b/yarn-project/noir-contracts.js/package.local.json index 0e363c59c36..9d7208eaa46 100644 --- a/yarn-project/noir-contracts.js/package.local.json +++ b/yarn-project/noir-contracts.js/package.local.json @@ -2,6 +2,6 @@ "scripts": { "build": "yarn clean && yarn build:contracts", "build:contracts": "./scripts/generate-types.sh", - "clean": "rm -rf .tsbuildinfo ./artifacts" + "clean": "rm -rf .tsbuildinfo ./artifacts ./codegenCache.json" } } \ No newline at end of file From ec1a99a80af3e786c20743740387800cc915c9e2 Mon Sep 17 00:00:00 2001 From: spypsy Date: Wed, 28 Feb 2024 13:10:00 +0000 Subject: [PATCH 6/8] do formatting separately to generating types --- yarn-project/noir-contracts.js/package.json | 4 ++-- yarn-project/noir-contracts.js/scripts/generate-types.sh | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/yarn-project/noir-contracts.js/package.json b/yarn-project/noir-contracts.js/package.json index b54febafcbb..8efbdbd1bc4 100644 --- a/yarn-project/noir-contracts.js/package.json +++ b/yarn-project/noir-contracts.js/package.json @@ -8,7 +8,7 @@ "./*": "./dest/src/*.js" }, "scripts": { - "build": "yarn clean && yarn build:contracts", + "build": "yarn clean && yarn build:contracts && yarn formatting:fix", "build:dev": "tsc -b --watch", "clean": "rm -rf .tsbuildinfo ./artifacts ./codegenCache.json", "formatting": "run -T prettier --check ./src && run -T eslint ./src", @@ -49,4 +49,4 @@ "engines": { "node": ">=18" } -} +} \ No newline at end of file diff --git a/yarn-project/noir-contracts.js/scripts/generate-types.sh b/yarn-project/noir-contracts.js/scripts/generate-types.sh index de07e0637ee..87a13d3459d 100755 --- a/yarn-project/noir-contracts.js/scripts/generate-types.sh +++ b/yarn-project/noir-contracts.js/scripts/generate-types.sh @@ -35,6 +35,3 @@ find "$OUT_DIR" -maxdepth 1 -type f -name '*.ts' ! -name 'index.ts' | while read CONTRACT_NAME=$(basename "$TS_FILE" .ts) # Remove the .ts extension to get the contract name echo "export * from './${CONTRACT_NAME}.js';" >>"$INDEX" done - -echo "Formatting..." -yarn formatting:fix From 8acd843ee6cb44cee44adbd8bd3d4a53d60d03d7 Mon Sep 17 00:00:00 2001 From: spypsy Date: Wed, 28 Feb 2024 13:55:55 +0000 Subject: [PATCH 7/8] update package.local.json --- yarn-project/noir-contracts.js/package.local.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/noir-contracts.js/package.local.json b/yarn-project/noir-contracts.js/package.local.json index 9d7208eaa46..ef0cd644072 100644 --- a/yarn-project/noir-contracts.js/package.local.json +++ b/yarn-project/noir-contracts.js/package.local.json @@ -1,6 +1,6 @@ { "scripts": { - "build": "yarn clean && yarn build:contracts", + "build": "yarn clean && yarn build:contracts && yarn formatting:fix", "build:contracts": "./scripts/generate-types.sh", "clean": "rm -rf .tsbuildinfo ./artifacts ./codegenCache.json" } From 767d5dbc411e5cf7032f07229224c1d6ee503707 Mon Sep 17 00:00:00 2001 From: spypsy Date: Wed, 28 Feb 2024 14:55:58 +0000 Subject: [PATCH 8/8] run yarn build in dockerfile too --- yarn-project/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/Dockerfile b/yarn-project/Dockerfile index 3349532fa20..d521bc9474c 100644 --- a/yarn-project/Dockerfile +++ b/yarn-project/Dockerfile @@ -18,7 +18,7 @@ RUN cd l1-artifacts && ./scripts/generate-artifacts.sh && rm -rf /usr/src/l1-con # This is actually our code generation tool. Needed to build contract typescript wrappers. RUN yarn workspace @aztec/noir-compiler build # Generates typescript wrappers. -RUN yarn workspace @aztec/noir-contracts.js build:contracts +RUN yarn workspace @aztec/noir-contracts.js build # We need to build accounts as it needs to copy in account contracts from noir-contracts. RUN yarn workspace @aztec/accounts build:copy-contracts RUN yarn workspace @aztec/protocol-contracts build:copy-contracts