feat: configure prover as separate process (#5973)

Update the `aztec` command to allow running as an independent prover
connected to a prover pool

yarn-project/aztec-node/src/aztec-node/config.ts

@@ -1,5 +1,6 @@
 import { type ArchiverConfig, getConfigEnvVars as getArchiverVars } from '@aztec/archiver';
 import { type P2PConfig, getP2PConfigEnvVars } from '@aztec/p2p';
+import { type ProverConfig, getProverEnvVars } from '@aztec/prover-client';
 import { type SequencerClientConfig, getConfigEnvVars as getSequencerVars } from '@aztec/sequencer-client';
 import { getConfigEnvVars as getWorldStateVars } from '@aztec/world-state';
 
@@ -8,6 +9,7 @@ import { getConfigEnvVars as getWorldStateVars } from '@aztec/world-state';
  */
 export type AztecNodeConfig = ArchiverConfig &
   SequencerClientConfig &
+  ProverConfig &
   P2PConfig & {
     /** Whether the sequencer is disabled for this node. */
     disableSequencer: boolean;
@@ -24,15 +26,17 @@ export type AztecNodeConfig = ArchiverConfig &
  * @returns A valid aztec node config.
  */
 export function getConfigEnvVars(): AztecNodeConfig {
-  const { SEQ_DISABLED, PROVER_DISABLED } = process.env;
+  const { SEQ_DISABLED, PROVER_DISABLED = '', ARCHIVER_URL } = process.env;
+
   const allEnvVars: AztecNodeConfig = {
     ...getSequencerVars(),
     ...getArchiverVars(),
     ...getP2PConfigEnvVars(),
     ...getWorldStateVars(),
+    ...getProverEnvVars(),
     disableSequencer: !!SEQ_DISABLED,
-    disableProver: !!PROVER_DISABLED,
-    archiverUrl: process.env.ARCHIVER_URL,
+    archiverUrl: ARCHIVER_URL,
+    disableProver: ['1', 'true'].includes(PROVER_DISABLED),
   };
 
   return allEnvVars;

yarn-project/aztec-node/src/aztec-node/server.ts

@@ -151,7 +151,7 @@ export class AztecNodeService implements AztecNode {
     const simulationProvider = await getSimulationProvider(config, log);
     const prover = config.disableProver
       ? await DummyProver.new()
-      : await TxProver.new(config, worldStateSynchronizer, simulationProvider);
+      : await TxProver.new(config, simulationProvider, worldStateSynchronizer);
 
     // now create the sequencer
     const sequencer = config.disableSequencer
@@ -194,6 +194,10 @@ export class AztecNodeService implements AztecNode {
     return this.sequencer;
   }
 
+  public getProver(): ProverClient {
+    return this.prover;
+  }
+
   /**
    * Method to return the currently deployed L1 contract addresses.
    * @returns - The currently deployed L1 contract addresses.

yarn-project/aztec/package.json

@@ -42,6 +42,7 @@
     "@aztec/noir-contracts.js": "workspace:^",
     "@aztec/p2p": "workspace:^",
     "@aztec/protocol-contracts": "workspace:^",
+    "@aztec/prover-client": "workspace:^",
     "@aztec/pxe": "workspace:^",
     "abitype": "^0.8.11",
     "commander": "^11.1.0",

yarn-project/aztec/src/cli/cli.ts

@@ -56,6 +56,9 @@ export function getProgram(userLog: LogFn, debugLogger: DebugLogger): Command {
       } else if (options.p2pBootstrap) {
         const { startP2PBootstrap } = await import('./cmds/start_p2p_bootstrap.js');
         await startP2PBootstrap(options, signalHandlers, userLog, debugLogger);
+      } else if (options.prover) {
+        const { startProver } = await import('./cmds/start_prover.js');
+        services = await startProver(options, signalHandlers, userLog);
       }
       if (services.length) {
         const rpcServer = createNamespacedJsonRpcServer(services, debugLogger);

yarn-project/aztec/src/cli/cmds/start_node.ts

@@ -6,6 +6,7 @@ import {
 import { NULL_KEY } from '@aztec/ethereum';
 import { type ServerList } from '@aztec/foundation/json-rpc/server';
 import { type LogFn } from '@aztec/foundation/log';
+import { createProvingJobSourceServer } from '@aztec/prover-client/prover-pool';
 import { type PXEServiceConfig, createPXERpcServer, getPXEServiceConfig } from '@aztec/pxe';
 
 import { mnemonicToAccount, privateKeyToAccount } from 'viem/accounts';
@@ -64,7 +65,10 @@ export const startNode = async (
   }
 
   if (!options.prover) {
+    userLog(`Prover is disabled, using mocked proofs`);
     nodeConfig.disableProver = true;
+  } else {
+    nodeConfig = mergeEnvVarsAndCliOptions<AztecNodeConfig>(nodeConfig, parseModuleOptions(options.prover));
   }
 
   if (!nodeConfig.disableSequencer && nodeConfig.disableProver) {
@@ -78,6 +82,11 @@ export const startNode = async (
   // Add node to services list
   services.push({ node: nodeServer });
 
+  if (!nodeConfig.disableProver) {
+    const provingJobSource = createProvingJobSourceServer(node.getProver().getProvingJobSource());
+    services.push({ provingJobSource });
+  }
+
   // Add node stop function to signal handlers
   signalHandlers.push(node.stop);
 

yarn-project/aztec/src/cli/cmds/start_prover.ts

@@ -0,0 +1,50 @@
+import { type ProvingJobSource } from '@aztec/circuit-types';
+import { ProverPool, createProvingJobSourceClient } from '@aztec/prover-client/prover-pool';
+
+import { type ServiceStarter, parseModuleOptions } from '../util.js';
+
+type ProverOptions = Partial<{
+  proverUrl: string;
+  agents: string;
+  acvmBinaryPath?: string;
+  bbBinaryPath?: string;
+  simulate?: string;
+}>;
+
+export const startProver: ServiceStarter = async (options, signalHandlers, logger) => {
+  const proverOptions: ProverOptions = parseModuleOptions(options.prover);
+  let source: ProvingJobSource;
+
+  if (typeof proverOptions.proverUrl === 'string') {
+    logger(`Connecting to prover at ${proverOptions.proverUrl}`);
+    source = createProvingJobSourceClient(proverOptions.proverUrl, 'provingJobSource');
+  } else {
+    throw new Error('Starting prover without an orchestrator is not supported');
+  }
+
+  const agentCount = proverOptions.agents ? parseInt(proverOptions.agents, 10) : 1;
+  if (agentCount === 0 || !Number.isSafeInteger(agentCount)) {
+    throw new Error('Cannot start prover without agents');
+  }
+
+  let pool: ProverPool;
+  if (proverOptions.simulate) {
+    pool = ProverPool.testPool(undefined, agentCount);
+  } else if (proverOptions.acvmBinaryPath && proverOptions.bbBinaryPath) {
+    pool = ProverPool.nativePool(
+      {
+        acvmBinaryPath: proverOptions.acvmBinaryPath,
+        bbBinaryPath: proverOptions.bbBinaryPath,
+      },
+      agentCount,
+    );
+  } else {
+    throw new Error('Cannot start prover without simulation or native prover options');
+  }
+
+  logger(`Starting ${agentCount} prover agents`);
+  await pool.start(source);
+  signalHandlers.push(() => pool.stop());
+
+  return Promise.resolve([]);
+};

yarn-project/aztec/src/cli/util.ts

@@ -3,10 +3,15 @@ import { type AztecNodeConfig } from '@aztec/aztec-node';
 import { type AccountManager, type Fr } from '@aztec/aztec.js';
 import { type L1ContractAddresses, l1ContractsNames } from '@aztec/ethereum';
 import { EthAddress } from '@aztec/foundation/eth-address';
+import { type ServerList } from '@aztec/foundation/json-rpc/server';
 import { type LogFn, createConsoleLogger } from '@aztec/foundation/log';
 import { type P2PConfig } from '@aztec/p2p';
 import { type PXEService, type PXEServiceConfig } from '@aztec/pxe';
 
+export interface ServiceStarter<T = any> {
+  (options: T, signalHandlers: (() => Promise<void>)[], logger: LogFn): Promise<ServerList>;
+}
+
 /**
  * Checks if the object has l1Contracts property
  * @param obj - The object to check

yarn-project/aztec/tsconfig.json

@@ -51,6 +51,9 @@
     {
       "path": "../protocol-contracts"
     },
+    {
+      "path": "../prover-client"
+    },
     {
       "path": "../pxe"
     }

yarn-project/circuit-types/src/interfaces/prover-client.ts

@@ -1,4 +1,5 @@
 import { type BlockProver } from './block-prover.js';
+import { type ProvingJobSource } from './proving-job.js';
 
 /**
  * The interface to the prover client.
@@ -8,4 +9,6 @@ export interface ProverClient extends BlockProver {
   start(): Promise<void>;
 
   stop(): Promise<void>;
+
+  getProvingJobSource(): ProvingJobSource;
 }

yarn-project/circuits.js/src/structs/kernel/kernel_circuit_public_inputs.ts

@@ -78,4 +78,12 @@ export class KernelCircuitPublicInputs {
       RevertCode.OK,
     );
   }
+
+  toString() {
+    return this.toBuffer().toString('hex');
+  }
+
+  static fromString(str: string) {
+    return KernelCircuitPublicInputs.fromBuffer(Buffer.from(str, 'hex'));
+  }
 }

yarn-project/circuits.js/src/structs/kernel/public_kernel_circuit_public_inputs.ts

@@ -51,6 +51,14 @@ export class PublicKernelCircuitPublicInputs {
     );
   }
 
+  toString() {
+    return this.toBuffer().toString('hex');
+  }
+
+  static fromString(str: string) {
+    return PublicKernelCircuitPublicInputs.fromBuffer(Buffer.from(str, 'hex'));
+  }
+
   get needsSetup() {
     return !this.endNonRevertibleData.publicCallStack[1].isEmpty();
   }

yarn-project/circuits.js/src/structs/kernel/public_kernel_tail_circuit_private_inputs.test.ts

@@ -14,4 +14,11 @@ describe('PublicKernelTailCircuitPrivateInputs', () => {
     expect(original).toEqual(serialized);
     expect(original).not.toBe(serialized);
   });
+
+  it('serializes to string and back', () => {
+    const original = makePublicKernelTailCircuitPrivateInputs(123);
+    const str = original.toString();
+    const deserialized = PublicKernelTailCircuitPrivateInputs.fromString(str);
+    expect(original).toEqual(deserialized);
+  });
 });

yarn-project/circuits.js/src/structs/kernel/public_kernel_tail_circuit_private_inputs.ts

@@ -41,6 +41,14 @@ export class PublicKernelTailCircuitPrivateInputs {
     );
   }
 
+  toString() {
+    return this.toBuffer().toString('hex');
+  }
+
+  static fromString(str: string) {
+    return PublicKernelTailCircuitPrivateInputs.fromBuffer(Buffer.from(str, 'hex'));
+  }
+
   static fromBuffer(buffer: Buffer | BufferReader) {
     const reader = BufferReader.asReader(buffer);
     return new PublicKernelTailCircuitPrivateInputs(

yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts

@@ -143,7 +143,7 @@ describe('L1Publisher integration', () => {
     };
     const worldStateSynchronizer = new ServerWorldStateSynchronizer(tmpStore, builderDb, blockSource, worldStateConfig);
     await worldStateSynchronizer.start();
-    builder = await TxProver.new({}, worldStateSynchronizer, new WASMSimulator());
+    builder = await TxProver.new(config, new WASMSimulator(), worldStateSynchronizer);
     l2Proof = Buffer.alloc(0);
 
     publisher = getL1Publisher({

yarn-project/prover-client/package.json

@@ -2,7 +2,10 @@
   "name": "@aztec/prover-client",
   "version": "0.1.0",
   "type": "module",
-  "exports": "./dest/index.js",
+  "exports": {
+    ".": "./dest/index.js",
+    "./prover-pool": "./dest/prover-pool/index.js"
+  },
   "bin": {
     "bb-cli": "./dest/bb/index.js"
   },

yarn-project/prover-client/src/config.ts

@@ -1,22 +1,47 @@
+import { tmpdir } from 'os';
+
 /**
  * The prover configuration.
  */
 export interface ProverConfig {
   /** The working directory to use for simulation/proving */
-  acvmWorkingDirectory?: string;
+  acvmWorkingDirectory: string;
   /** The path to the ACVM binary */
-  acvmBinaryPath?: string;
+  acvmBinaryPath: string;
+  /** The working directory to for proving */
+  bbWorkingDirectory: string;
+  /** The path to the bb binary */
+  bbBinaryPath: string;
+  /** How many agents to start */
+  proverAgents: number;
+  /** Enable proving. If true, must set bb env vars */
+  realProofs: boolean;
 }
 
 /**
  * Returns the prover configuration from the environment variables.
  * Note: If an environment variable is not set, the default value is used.
  * @returns The prover configuration.
  */
-export function getConfigEnvVars(): ProverConfig {
-  const { ACVM_WORKING_DIRECTORY, ACVM_BINARY_PATH } = process.env;
+export function getProverEnvVars(): ProverConfig {
+  const {
+    ACVM_WORKING_DIRECTORY = tmpdir(),
+    ACVM_BINARY_PATH = '',
+    BB_WORKING_DIRECTORY = tmpdir(),
+    BB_BINARY_PATH = '',
+    PROVER_AGENTS = '1',
+    PROVER_REAL_PROOFS = '',
+  } = process.env;
+
+  const parsedProverAgents = parseInt(PROVER_AGENTS, 10);
+  const proverAgents = Number.isSafeInteger(parsedProverAgents) ? parsedProverAgents : 0;
+
   return {
-    acvmWorkingDirectory: ACVM_WORKING_DIRECTORY ? ACVM_WORKING_DIRECTORY : undefined,
-    acvmBinaryPath: ACVM_BINARY_PATH ? ACVM_BINARY_PATH : undefined,
+    acvmWorkingDirectory: ACVM_WORKING_DIRECTORY,
+    acvmBinaryPath: ACVM_BINARY_PATH,
+    bbBinaryPath: BB_BINARY_PATH,
+    bbWorkingDirectory: BB_WORKING_DIRECTORY,
+    proverAgents,
+    realProofs: ['1', 'true'].includes(PROVER_REAL_PROOFS),
   };
 }

yarn-project/prover-client/src/dummy-prover.ts

@@ -4,13 +4,18 @@ import {
   PROVING_STATUS,
   type ProcessedTx,
   type ProverClient,
+  type ProvingJob,
+  type ProvingJobSource,
+  type ProvingRequest,
   type ProvingSuccess,
   type ProvingTicket,
 } from '@aztec/circuit-types';
 import { type GlobalVariables, makeEmptyProof } from '@aztec/circuits.js';
 import { type Fr } from '@aztec/foundation/fields';
 
 export class DummyProver implements ProverClient {
+  jobs = new DummyProvingJobSource();
+
   public start(): Promise<void> {
     return Promise.resolve();
   }
@@ -54,4 +59,22 @@ export class DummyProver implements ProverClient {
   setBlockCompleted(): Promise<void> {
     return Promise.resolve();
   }
+
+  getProvingJobSource(): ProvingJobSource {
+    return this.jobs;
+  }
+}
+
+class DummyProvingJobSource implements ProvingJobSource {
+  getProvingJob(): Promise<ProvingJob<ProvingRequest> | null> {
+    return Promise.resolve(null);
+  }
+
+  rejectProvingJob(): Promise<void> {
+    return Promise.resolve();
+  }
+
+  resolveProvingJob(): Promise<void> {
+    return Promise.resolve();
+  }
 }

yarn-project/prover-client/src/prover-pool/index.ts

@@ -1,3 +1,4 @@
 export * from './prover-agent.js';
 export * from './memory-proving-queue.js';
 export * from './prover-pool.js';
+export * from './rpc.js';

yarn-project/prover-client/src/prover-pool/prover-agent.ts

@@ -11,6 +11,7 @@ import { RunningPromise } from '@aztec/foundation/running-promise';
 import { elapsed } from '@aztec/foundation/timer';
 
 import { type CircuitProver } from '../prover/interface.js';
+import { ProvingError } from './proving-error.js';
 
 export class ProverAgent {
   private runningPromise?: RunningPromise;
@@ -46,7 +47,7 @@ export class ProverAgent {
         this.log.error(
           `Error processing proving job id=${job.id} type=${ProvingRequestType[job.request.type]}: ${err}`,
         );
-        await queue.rejectProvingJob(job.id, err as Error);
+        await queue.rejectProvingJob(job.id, new ProvingError((err as any)?.message ?? String(err)));
       }
     }, this.intervalMs);