chore: Optimise bb.js package size and sandox/cli dockerfiles to unbl…

…oat final containers. (#3462)

We're making efforts to streamline our container sizes. They shouldn't
really be any "bigger" than an equivalent npm download and package
install.

* bb.js inlined the two wasms creating a 50mb monster index.js because:
* The wasm symbol stripping was not being done after wasm creation in
the bb wasm container.
* The two wasms ended up duplicated in the bundle due to webpack inline
the workers, and seemingly unable to reason they're the same thing even
using chunks.
* Given that bb.js now has a synchronous singleton instantiation of the
wasm, we can plumb that compiled module into the async worker version to
avoid the duplication.
* The above fixes reduce the size of index.js to 11mb, and this
compresses very nicely to ~1.7mb for serving over web.
* We still produce a "runnable" container in bb.js, for executing the
tests, but we use `yarn pack` to produce the publishable artefact, which
is what we import downstream. This cuts out 300mb of node_modules, which
may just get reimported downstream, but should at least remove
duplication and strip back to production on dependencies.
* We remove gubbins from bb wasm container that circuits cpp needed, as
we have killed it off.
* For `sandbox` and `cli` we:
* Fix the "productionify" step to clear the cache after to the prod
install. It was originally this way around due to use of pnp instead of
node_modules (if we really care about size, maybe we should reconsider
pnpifying the final runtime?). But, if we're not using pnp, we should
purge the `.yarn/cache`.
  * Blow away all the `src` folders as they're non negligible in size.
* Building final "slim" image cherry picks a little more carefully to
avoid pulling in some unneeding things (e.g. nargo).

barretenberg/cpp/.dockerignore

@@ -17,5 +17,6 @@
 
 # Needed scripts.
 !scripts/install-wasi-sdk.sh
+!scripts/strip-wasm.sh
 !./.clang-format
 !./format.sh

barretenberg/cpp/dockerfiles/Dockerfile.wasm-linux-clang

@@ -6,15 +6,10 @@ RUN ./scripts/install-wasi-sdk.sh
 COPY . .
 RUN cmake --preset wasm && cmake --build --preset wasm
 RUN cmake --preset wasm-threads && cmake --build --preset wasm-threads
+RUN ./scripts/strip-wasm.sh
 
 FROM scratch
 WORKDIR /usr/src/barretenberg/cpp
 COPY . .
 COPY --from=builder /usr/src/barretenberg/cpp/build-wasm/bin/barretenberg.wasm /usr/src/barretenberg/cpp/build-wasm/bin/barretenberg.wasm
 COPY --from=builder /usr/src/barretenberg/cpp/build-wasm-threads/bin/barretenberg.wasm /usr/src/barretenberg/cpp/build-wasm-threads/bin/barretenberg.wasm
-# Copy libs for consuming projects.
-COPY --from=builder /usr/src/barretenberg/cpp/build-wasm/lib/libbarretenberg.a /usr/src/barretenberg/cpp/build-wasm/lib/libbarretenberg.a
-COPY --from=builder /usr/src/barretenberg/cpp/build-wasm/lib/libwasi.a /usr/src/barretenberg/cpp/build-wasm/lib/libwasi.a
-COPY --from=builder /usr/src/barretenberg/cpp/build-wasm/lib/libenv.a /usr/src/barretenberg/cpp/build-wasm/lib/libenv.a
-# Copy wasi-sdk so that consuming projects have the toolchain available.
-COPY --from=builder /usr/src/barretenberg/cpp/src/wasi-sdk-20.0 /usr/src/barretenberg/cpp/src/wasi-sdk-20.0

barretenberg/ts/.gitignore

@@ -5,3 +5,5 @@ dest
 .tsbuildinfo*
 *.log
 /crs
+package.tgz
+package

barretenberg/ts/Dockerfile

@@ -2,6 +2,9 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/barretenberg-wasm-linux-clang
 
 FROM node:18-alpine
 COPY --from=0 /usr/src/barretenberg /usr/src/barretenberg
+
+# Create a standalone container that can run bb.js (and tests).
+# We'll perform the build in a new, different directory, so the original directory can become the "published" package.
 WORKDIR /usr/src/barretenberg/ts
 # Leverage layer caching. Only re-install packages if these files change.
 COPY .yarn .yarn
@@ -12,3 +15,6 @@ RUN yarn --immutable
 COPY . .
 RUN yarn formatting && SKIP_CPP_BUILD=1 yarn build
 CMD ["yarn", "test"]
+
+# We want to create a pure package, as would be published to npm, for consuming projects.
+RUN yarn pack && tar zxf package.tgz && rm package.tgz

barretenberg/ts/package.json

@@ -20,19 +20,19 @@
     "README.md"
   ],
   "scripts": {
-    "clean": "rm -rf ./dest .tsbuildinfo .tsbuildinfo.cjs",
-    "build": "yarn clean && yarn build:wasm && yarn build:esm && yarn build:cjs && yarn build:browser",
+    "clean": "rm -rf ./dest .tsbuildinfo .tsbuildinfo.cjs package.tgz package",
+    "build": "yarn clean && yarn build:wasm && yarn build:esm && yarn build:cjs && yarn build:browser && yarn build:package",
     "build:wasm": "./scripts/build_wasm.sh",
     "build:esm": "tsc -b && chmod +x ./dest/node/main.js",
     "build:cjs": "tsc -b tsconfig.cjs.json && ./scripts/cjs_postprocess.sh",
     "build:browser": "webpack",
     "build:bindings": "cd .. && ./scripts/bindgen.sh",
+    "build:package": "yarn pack && tar zxf package.tgz && rm -f package.tgz",
     "formatting": "prettier --check ./src && eslint --max-warnings 0 ./src",
     "formatting:fix": "prettier -w ./src",
     "test": "NODE_OPTIONS='--loader ts-node/esm' NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --no-cache --passWithNoTests",
     "test:debug": "NODE_OPTIONS='--loader ts-node/esm' NODE_NO_WARNINGS=1 node --inspect-brk=0.0.0.0 --experimental-vm-modules $(yarn bin jest) --no-cache --passWithNoTests --runInBand",
     "simple_test": "NODE_OPTIONS='--loader ts-node/esm' NODE_NO_WARNINGS=1 node ./src/examples/simple.rawtest.ts",
-    "prepack": "yarn build",
     "deploy": "npm publish --access public"
   },
   "jest": {

barretenberg/ts/src/barretenberg/index.ts

@@ -3,7 +3,7 @@ import { BarretenbergApi, BarretenbergApiSync } from '../barretenberg_api/index.
 import { createMainWorker } from '../barretenberg_wasm/barretenberg_wasm_main/factory/node/index.js';
 import { BarretenbergWasmMain, BarretenbergWasmMainWorker } from '../barretenberg_wasm/barretenberg_wasm_main/index.js';
 import { getRemoteBarretenbergWasm } from '../barretenberg_wasm/helpers/index.js';
-import { BarretenbergWasmWorker } from '../barretenberg_wasm/index.js';
+import { BarretenbergWasmWorker, fetchModuleAndThreads } from '../barretenberg_wasm/index.js';
 import createDebug from 'debug';
 
 const debug = createDebug('bb.js:wasm');
@@ -19,14 +19,15 @@ export class Barretenberg extends BarretenbergApi {
 
   /**
    * Constructs an instance of Barretenberg.
-   * Launches it within a worker. This is necessary as it block waiting on child threads to complete,
+   * Launches it within a worker. This is necessary as it blocks waiting on child threads to complete,
    * and blocking the main thread in the browser is not allowed.
    * It threads > 1 (defaults to hardware availability), child threads will be created on their own workers.
    */
-  static async new(threads?: number) {
+  static async new(desiredThreads?: number) {
     const worker = createMainWorker();
     const wasm = getRemoteBarretenbergWasm<BarretenbergWasmMainWorker>(worker);
-    await wasm.init(threads, proxy(debug));
+    const { module, threads } = await fetchModuleAndThreads(desiredThreads);
+    await wasm.init(module, threads, proxy(debug));
     return new Barretenberg(worker, wasm);
   }
 
@@ -49,7 +50,8 @@ export class BarretenbergSync extends BarretenbergApiSync {
 
   static async new() {
     const wasm = new BarretenbergWasmMain();
-    await wasm.init(1);
+    const { module, threads } = await fetchModuleAndThreads(1);
+    await wasm.init(module, threads);
     return new BarretenbergSync(wasm);
   }
 

barretenberg/ts/src/barretenberg_wasm/barretenberg_wasm_main/index.ts

@@ -2,7 +2,6 @@ import { type Worker } from 'worker_threads';
 import createDebug from 'debug';
 import { Remote } from 'comlink';
 import { getNumCpu, getRemoteBarretenbergWasm, getSharedMemoryAvailable } from '../helpers/index.js';
-import { fetchCode } from '../fetch_code/index.js';
 import { createThreadWorker } from '../barretenberg_wasm_thread/factory/node/index.js';
 import { type BarretenbergWasmThreadWorker } from '../barretenberg_wasm_thread/index.js';
 import { BarretenbergWasmBase } from '../barretenberg_wasm_base/index.js';
@@ -30,6 +29,7 @@ export class BarretenbergWasmMain extends BarretenbergWasmBase {
    * Init as main thread. Spawn child threads.
    */
   public async init(
+    module: WebAssembly.Module,
     threads = Math.min(getNumCpu(), BarretenbergWasmMain.MAX_THREADS),
     logger: (msg: string) => void = debug,
     initial = 25,
@@ -41,10 +41,6 @@ export class BarretenbergWasmMain extends BarretenbergWasmBase {
     const maxMb = (maximum * 2 ** 16) / (1024 * 1024);
     const shared = getSharedMemoryAvailable();
 
-    if (!shared) {
-      threads = 1;
-    }
-
     this.logger(
       `initial mem: ${initial} pages, ${initialMb}MiB. ` +
         `max mem: ${maximum} pages, ${maxMb}MiB. ` +
@@ -53,8 +49,7 @@ export class BarretenbergWasmMain extends BarretenbergWasmBase {
 
     this.memory = new WebAssembly.Memory({ initial, maximum, shared });
 
-    const code = await fetchCode(shared);
-    const { instance, module } = await WebAssembly.instantiate(code, this.getImportObj(this.memory));
+    const instance = await WebAssembly.instantiate(module, this.getImportObj(this.memory));
 
     this.instance = instance;
 

barretenberg/ts/src/barretenberg_wasm/helpers/node/index.ts

@@ -17,8 +17,11 @@ export function getRemoteBarretenbergWasm<T>(worker: Worker): T {
   return wrap(nodeEndpoint(worker)) as T;
 }
 
+/**
+ * Returns number of cpus as reported by the system, unless overriden by HARDWARE_CONCURRENCY env var.
+ */
 export function getNumCpu() {
-  return os.cpus().length;
+  return +process.env.HARDWARE_CONCURRENCY! || os.cpus().length;
 }
 
 /**

barretenberg/ts/src/barretenberg_wasm/index.ts

@@ -1,20 +1,30 @@
 import { proxy } from 'comlink';
 import createDebug from 'debug';
 import { createMainWorker } from './barretenberg_wasm_main/factory/node/index.js';
-import { getRemoteBarretenbergWasm } from './helpers/node/index.js';
+import { getRemoteBarretenbergWasm, getSharedMemoryAvailable } from './helpers/node/index.js';
 import { BarretenbergWasmMain, BarretenbergWasmMainWorker } from './barretenberg_wasm_main/index.js';
+import { fetchCode } from './fetch_code/index.js';
 
 const debug = createDebug('bb.js:wasm');
 
+export async function fetchModuleAndThreads(desiredThreads?: number) {
+  const shared = getSharedMemoryAvailable();
+  const threads = shared ? desiredThreads : 1;
+  const code = await fetchCode(shared);
+  const module = await WebAssembly.compile(code);
+  return { module, threads };
+}
+
 export class BarretenbergWasm extends BarretenbergWasmMain {
   /**
    * Construct and initialize BarretenbergWasm within a Worker. Return both the worker and the wasm proxy.
    * Used when running in the browser, because we can't block the main thread.
    */
-  public static async new(threads?: number) {
+  public static async new(desiredThreads?: number) {
     const worker = createMainWorker();
     const wasm = getRemoteBarretenbergWasm<BarretenbergWasmMainWorker>(worker);
-    await wasm.init(threads, proxy(debug));
+    const { module, threads } = await fetchModuleAndThreads(desiredThreads);
+    await wasm.init(module, threads, proxy(debug));
     return { worker, wasm };
   }
 }