From 1db852e930907efa7f150eaca9633c2e8b72e977 Mon Sep 17 00:00:00 2001 From: Gregor Date: Fri, 16 Jun 2023 17:46:22 +0200 Subject: [PATCH 1/3] separate type checking from build script --- js/node/node-backend.js | 5 +- js/wrapper.js | 5 +- scripts/build-snarkyjs-node-artifacts.sh | 66 ++++++++++++++++++++++++ scripts/build-snarkyjs-node.sh | 63 +--------------------- scripts/update-snarkyjs-bindings.sh | 2 +- 5 files changed, 77 insertions(+), 64 deletions(-) create mode 100755 scripts/build-snarkyjs-node-artifacts.sh diff --git a/js/node/node-backend.js b/js/node/node-backend.js index bb6fc61b..f1106dae 100644 --- a/js/node/node-backend.js +++ b/js/node/node-backend.js @@ -1,6 +1,9 @@ import { isMainThread, parentPort, workerData, Worker } from 'worker_threads'; import os from 'os'; -import wasm_ from '../../../_node_bindings/plonk_wasm.cjs'; +// TODO: the modified directory _node_bindings (with underscore) is a hack to +// prevent typescript from processing this folder. +// any cleaner solution is appreciated! +import wasm_ from '../../compiled/_node_bindings/plonk_wasm.cjs'; import { fileURLToPath } from 'url'; let url = import.meta.url; let filename = url !== undefined ? fileURLToPath(url) : __filename; diff --git a/js/wrapper.js b/js/wrapper.js index 50433a63..a0ff4f36 100644 --- a/js/wrapper.js +++ b/js/wrapper.js @@ -1,5 +1,8 @@ import { wasm, withThreadPool } from './node/node-backend.js'; -import { default as snarky } from '../../_node_bindings/snarky_js_node.bc.cjs'; +// TODO: the modified directory _node_bindings (with underscore) is a hack to +// prevent typescript from processing this folder. +// any cleaner solution is appreciated! +import { default as snarky } from '../compiled/_node_bindings/snarky_js_node.bc.cjs'; export { getSnarky, getWasm, withThreadPool }; diff --git a/scripts/build-snarkyjs-node-artifacts.sh b/scripts/build-snarkyjs-node-artifacts.sh new file mode 100755 index 00000000..62b81cae --- /dev/null +++ b/scripts/build-snarkyjs-node-artifacts.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +set -e + +SNARKY_JS_PATH="src/lib/snarkyjs" +DUNE_PATH="$SNARKY_JS_PATH/src/bindings/ocaml" +BUILD_PATH="_build/default/$DUNE_PATH" +KIMCHI_BINDINGS="$SNARKY_JS_PATH/src/bindings/kimchi" + +pushd "$SNARKY_JS_PATH" + [ -d node_modules ] || npm i +popd + +export DUNE_USE_DEFAULT_LINKER="y" + +if [ -f "$BUILD_PATH/snarky_js_node.bc.js" ]; then + echo "found snarky_js_node.bc.js" + if [ -f "$BUILD_PATH/snarky_js_node.bc.map" ]; then + echo "found snarky_js_node.bc.map, saving at a tmp location because dune will delete it" + cp "$BUILD_PATH/snarky_js_node.bc.map" _build/snarky_js_node.bc.map ; + else + echo "did not find snarky_js_node.bc.map, deleting snarky_js_node.bc.js to force calling jsoo again" + rm -f "$BUILD_PATH/snarky_js_node.bc.js" + fi +fi + +dune b $KIMCHI_BINDINGS/js/node_js \ +&& dune b $DUNE_PATH/snarky_js_node.bc.js || exit 1 + +# update if new source map was built +if [ -f "$BUILD_PATH/snarky_js_node.bc.map" ]; then + echo "new source map created"; + cp "$BUILD_PATH/snarky_js_node.bc.map" "_build/snarky_js_node.bc.map"; +fi + +dune b $SNARKY_JS_PATH/src/bindings/mina-transaction/gen/js-layout.ts \ +&& dune b $SNARKY_JS_PATH/src/bindings/crypto/constants.ts \ + $SNARKY_JS_PATH/src/bindings/crypto/test_vectors/poseidonKimchi.ts \ +|| exit 1 + +BINDINGS_PATH="$SNARKY_JS_PATH"/dist/node/bindings/compiled/_node_bindings/ +mkdir -p "$BINDINGS_PATH" +chmod -R 777 "$BINDINGS_PATH" +cp _build/default/$KIMCHI_BINDINGS/js/node_js/plonk_wasm* "$BINDINGS_PATH" +mv -f $BINDINGS_PATH/plonk_wasm.js $BINDINGS_PATH/plonk_wasm.cjs +cp $BUILD_PATH/snarky_js_node*.js "$BINDINGS_PATH" +cp "_build/snarky_js_node.bc.map" "$BINDINGS_PATH"/snarky_js_node.bc.map +mv -f $BINDINGS_PATH/snarky_js_node.bc.js $BINDINGS_PATH/snarky_js_node.bc.cjs +sed -i 's/plonk_wasm.js/plonk_wasm.cjs/' $BINDINGS_PATH/snarky_js_node.bc.cjs + +# cleanup tmp source map +cp _build/snarky_js_node.bc.map $BUILD_PATH/snarky_js_node.bc.map +rm -f _build/snarky_js_node.bc.map + +# better error messages +# TODO: find a less hacky way to make adjustments to jsoo compiler output +# `s` is the jsoo representation of the error message string, and `s.c` is the actual JS string +sed -i 's/function failwith(s){throw \[0,Failure,s\]/function failwith(s){throw joo_global_object.Error(s.c)/' "$BINDINGS_PATH"/snarky_js_node.bc.cjs +sed -i 's/function invalid_arg(s){throw \[0,Invalid_argument,s\]/function invalid_arg(s){throw joo_global_object.Error(s.c)/' "$BINDINGS_PATH"/snarky_js_node.bc.cjs +sed -i 's/return \[0,Exn,t\]/return joo_global_object.Error(t.c)/' "$BINDINGS_PATH"/snarky_js_node.bc.cjs +# TODO: this doesn't cover all cases, maybe should rewrite to_exn instead +sed -i 's/function raise(t){throw caml_call1(to_exn$0,t)}/function raise(t){throw Error(t?.[1]?.c ?? "Unknown error thrown by raise")}/' "$BINDINGS_PATH"/snarky_js_node.bc.cjs + +chmod 777 "$BINDINGS_PATH"/* +node "$SNARKY_JS_PATH/src/build/fix-wasm-bindings-node.js" "$BINDINGS_PATH/plonk_wasm.cjs" + diff --git a/scripts/build-snarkyjs-node.sh b/scripts/build-snarkyjs-node.sh index a0322a44..4f8d7746 100755 --- a/scripts/build-snarkyjs-node.sh +++ b/scripts/build-snarkyjs-node.sh @@ -2,66 +2,7 @@ set -e -SNARKY_JS_PATH="src/lib/snarkyjs" -DUNE_PATH="$SNARKY_JS_PATH/src/bindings/ocaml" -BUILD_PATH="_build/default/$DUNE_PATH" -KIMCHI_BINDINGS="$SNARKY_JS_PATH/src/bindings/kimchi" - -pushd "$SNARKY_JS_PATH" - [ -d node_modules ] || npm i -popd - -export DUNE_USE_DEFAULT_LINKER="y" - -if [ -f "$BUILD_PATH/snarky_js_node.bc.js" ]; then - echo "found snarky_js_node.bc.js" - if [ -f "$BUILD_PATH/snarky_js_node.bc.map" ]; then - echo "found snarky_js_node.bc.map, saving at a tmp location because dune will delete it" - cp "$BUILD_PATH/snarky_js_node.bc.map" _build/snarky_js_node.bc.map ; - else - echo "did not find snarky_js_node.bc.map, deleting snarky_js_node.bc.js to force calling jsoo again" - rm -f "$BUILD_PATH/snarky_js_node.bc.js" - fi -fi - -dune b $KIMCHI_BINDINGS/js/node_js \ -&& dune b $DUNE_PATH/snarky_js_node.bc.js || exit 1 - -# update if new source map was built -if [ -f "$BUILD_PATH/snarky_js_node.bc.map" ]; then - echo "new source map created"; - cp "$BUILD_PATH/snarky_js_node.bc.map" "_build/snarky_js_node.bc.map"; -fi - -dune b $SNARKY_JS_PATH/src/bindings/mina-transaction/gen/js-layout.ts \ -&& dune b $SNARKY_JS_PATH/src/bindings/crypto/constants.ts \ - $SNARKY_JS_PATH/src/bindings/crypto/test_vectors/poseidonKimchi.ts \ -|| exit 1 - -BINDINGS_PATH="$SNARKY_JS_PATH"/dist/node/_node_bindings/ -mkdir -p "$BINDINGS_PATH" -chmod -R 777 "$BINDINGS_PATH" -cp _build/default/$KIMCHI_BINDINGS/js/node_js/plonk_wasm* "$BINDINGS_PATH" -mv -f $BINDINGS_PATH/plonk_wasm.js $BINDINGS_PATH/plonk_wasm.cjs -cp $BUILD_PATH/snarky_js_node*.js "$BINDINGS_PATH" -cp "_build/snarky_js_node.bc.map" "$BINDINGS_PATH"/snarky_js_node.bc.map -mv -f $BINDINGS_PATH/snarky_js_node.bc.js $BINDINGS_PATH/snarky_js_node.bc.cjs -sed -i 's/plonk_wasm.js/plonk_wasm.cjs/' $BINDINGS_PATH/snarky_js_node.bc.cjs - -# cleanup tmp source map -cp _build/snarky_js_node.bc.map $BUILD_PATH/snarky_js_node.bc.map -rm -f _build/snarky_js_node.bc.map - -# better error messages -# TODO: find a less hacky way to make adjustments to jsoo compiler output -# `s` is the jsoo representation of the error message string, and `s.c` is the actual JS string -sed -i 's/function failwith(s){throw \[0,Failure,s\]/function failwith(s){throw joo_global_object.Error(s.c)/' "$BINDINGS_PATH"/snarky_js_node.bc.cjs -sed -i 's/function invalid_arg(s){throw \[0,Invalid_argument,s\]/function invalid_arg(s){throw joo_global_object.Error(s.c)/' "$BINDINGS_PATH"/snarky_js_node.bc.cjs -sed -i 's/return \[0,Exn,t\]/return joo_global_object.Error(t.c)/' "$BINDINGS_PATH"/snarky_js_node.bc.cjs -# TODO: this doesn't cover all cases, maybe should rewrite to_exn instead -sed -i 's/function raise(t){throw caml_call1(to_exn$0,t)}/function raise(t){throw Error(t?.[1]?.c ?? "Unknown error thrown by raise")}/' "$BINDINGS_PATH"/snarky_js_node.bc.cjs - -chmod 777 "$BINDINGS_PATH"/* -node "$SNARKY_JS_PATH/src/build/fix-wasm-bindings-node.js" "$BINDINGS_PATH/plonk_wasm.cjs" +DIR_PATH=$(dirname "$0") +./${DIR_PATH}/build-snarkyjs-node-artifacts.sh npm run --prefix="$SNARKY_JS_PATH" dev diff --git a/scripts/update-snarkyjs-bindings.sh b/scripts/update-snarkyjs-bindings.sh index 6304c8d0..8230ea2e 100755 --- a/scripts/update-snarkyjs-bindings.sh +++ b/scripts/update-snarkyjs-bindings.sh @@ -16,7 +16,7 @@ $DIR_PATH/build-snarkyjs-node.sh chmod -R 777 "$NODE_BINDINGS" -BINDINGS_PATH="$SNARKY_JS_PATH"/dist/node/_node_bindings/ +BINDINGS_PATH="$SNARKY_JS_PATH"/dist/node/bindings/compiled/_node_bindings/ cp "$BINDINGS_PATH"/snarky_js_node.bc.cjs "$NODE_BINDINGS"/snarky_js_node.bc.cjs cp "$BINDINGS_PATH"/snarky_js_node.bc.map "$NODE_BINDINGS"/snarky_js_node.bc.map cp "$BINDINGS_PATH"/plonk_wasm* "$NODE_BINDINGS"/ From 4ac0036e75c3868f75d81ae9f1729096e451fbd0 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 20 Jun 2023 11:04:50 +0200 Subject: [PATCH 2/3] fixup --- scripts/build-snarkyjs-node.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/build-snarkyjs-node.sh b/scripts/build-snarkyjs-node.sh index 4f8d7746..1e16b32f 100755 --- a/scripts/build-snarkyjs-node.sh +++ b/scripts/build-snarkyjs-node.sh @@ -2,7 +2,6 @@ set -e -DIR_PATH=$(dirname "$0") +$(dirname "$0")/build-snarkyjs-node-artifacts.sh -./${DIR_PATH}/build-snarkyjs-node-artifacts.sh -npm run --prefix="$SNARKY_JS_PATH" dev +npm run --prefix="src/lib/snarkyjs" dev From 0202843678471768ea70b53f21306186f782677c Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 20 Jun 2023 20:35:23 +0200 Subject: [PATCH 3/3] move bindings location and add guarding d.ts --- compiled/.gitignore | 1 + compiled/node_bindings/{plonk_wasm.d.ts => plonk_wasm.d.cts} | 0 compiled/node_bindings/snarky_js_node.bc.d.cts | 1 + js/node/node-backend.js | 3 --- js/wrapper.d.ts | 2 +- js/wrapper.js | 3 --- scripts/build-snarkyjs-node-artifacts.sh | 4 +++- 7 files changed, 6 insertions(+), 8 deletions(-) create mode 100644 compiled/.gitignore rename compiled/node_bindings/{plonk_wasm.d.ts => plonk_wasm.d.cts} (100%) create mode 100644 compiled/node_bindings/snarky_js_node.bc.d.cts diff --git a/compiled/.gitignore b/compiled/.gitignore new file mode 100644 index 00000000..6f356acb --- /dev/null +++ b/compiled/.gitignore @@ -0,0 +1 @@ +_node_bindings diff --git a/compiled/node_bindings/plonk_wasm.d.ts b/compiled/node_bindings/plonk_wasm.d.cts similarity index 100% rename from compiled/node_bindings/plonk_wasm.d.ts rename to compiled/node_bindings/plonk_wasm.d.cts diff --git a/compiled/node_bindings/snarky_js_node.bc.d.cts b/compiled/node_bindings/snarky_js_node.bc.d.cts new file mode 100644 index 00000000..7473aeb9 --- /dev/null +++ b/compiled/node_bindings/snarky_js_node.bc.d.cts @@ -0,0 +1 @@ +// this file exists to prevent TS from type-checking `snarky_js_node.bc.cjs` diff --git a/js/node/node-backend.js b/js/node/node-backend.js index f1106dae..d1ac1943 100644 --- a/js/node/node-backend.js +++ b/js/node/node-backend.js @@ -1,8 +1,5 @@ import { isMainThread, parentPort, workerData, Worker } from 'worker_threads'; import os from 'os'; -// TODO: the modified directory _node_bindings (with underscore) is a hack to -// prevent typescript from processing this folder. -// any cleaner solution is appreciated! import wasm_ from '../../compiled/_node_bindings/plonk_wasm.cjs'; import { fileURLToPath } from 'url'; let url = import.meta.url; diff --git a/js/wrapper.d.ts b/js/wrapper.d.ts index 5f903fb5..e850b743 100644 --- a/js/wrapper.d.ts +++ b/js/wrapper.d.ts @@ -1,4 +1,4 @@ -import * as wasm from '../compiled/node_bindings/plonk_wasm.js'; +import * as wasm from '../compiled/node_bindings/plonk_wasm.cjs'; export { WasmModule, getWasm, getSnarky, withThreadPool }; diff --git a/js/wrapper.js b/js/wrapper.js index a0ff4f36..9ad9ce9d 100644 --- a/js/wrapper.js +++ b/js/wrapper.js @@ -1,7 +1,4 @@ import { wasm, withThreadPool } from './node/node-backend.js'; -// TODO: the modified directory _node_bindings (with underscore) is a hack to -// prevent typescript from processing this folder. -// any cleaner solution is appreciated! import { default as snarky } from '../compiled/_node_bindings/snarky_js_node.bc.cjs'; export { getSnarky, getWasm, withThreadPool }; diff --git a/scripts/build-snarkyjs-node-artifacts.sh b/scripts/build-snarkyjs-node-artifacts.sh index 62b81cae..6073bcf9 100755 --- a/scripts/build-snarkyjs-node-artifacts.sh +++ b/scripts/build-snarkyjs-node-artifacts.sh @@ -38,12 +38,14 @@ dune b $SNARKY_JS_PATH/src/bindings/mina-transaction/gen/js-layout.ts \ $SNARKY_JS_PATH/src/bindings/crypto/test_vectors/poseidonKimchi.ts \ || exit 1 -BINDINGS_PATH="$SNARKY_JS_PATH"/dist/node/bindings/compiled/_node_bindings/ +BINDINGS_PATH="$SNARKY_JS_PATH"/src/bindings/compiled/_node_bindings/ mkdir -p "$BINDINGS_PATH" chmod -R 777 "$BINDINGS_PATH" cp _build/default/$KIMCHI_BINDINGS/js/node_js/plonk_wasm* "$BINDINGS_PATH" mv -f $BINDINGS_PATH/plonk_wasm.js $BINDINGS_PATH/plonk_wasm.cjs +mv -f $BINDINGS_PATH/plonk_wasm.d.ts $BINDINGS_PATH/plonk_wasm.d.cts cp $BUILD_PATH/snarky_js_node*.js "$BINDINGS_PATH" +cp $SNARKY_JS_PATH/src/bindings/compiled/node_bindings/snarky_js_node.bc.d.cts $BINDINGS_PATH/ cp "_build/snarky_js_node.bc.map" "$BINDINGS_PATH"/snarky_js_node.bc.map mv -f $BINDINGS_PATH/snarky_js_node.bc.js $BINDINGS_PATH/snarky_js_node.bc.cjs sed -i 's/plonk_wasm.js/plonk_wasm.cjs/' $BINDINGS_PATH/snarky_js_node.bc.cjs