feat(cli): declarative deployment (#1702)

Co-authored-by: alvarius <alvarius@lattice.xyz>

.changeset/poor-bags-stare.md

@@ -0,0 +1,14 @@
+---
+"@latticexyz/cli": major
+---
+
+`deploy`, `test`, `dev-contracts` were overhauled using a declarative deployment approach under the hood. Deploys are now idempotent and re-running them will introspect the world and figure out the minimal changes necessary to bring the world into alignment with its config: adding tables, adding/upgrading systems, changing access control, etc.
+
+The following CLI arguments are now removed from these commands:
+
+- `--debug` (you can now adjust CLI output with `DEBUG` environment variable, e.g. `DEBUG=mud:*`)
+- `--priorityFeeMultiplier` (now calculated automatically)
+- `--disableTxWait` (everything is now parallelized with smarter nonce management)
+- `--pollInterval` (we now lean on viem defaults and we don't wait/poll until the very end of the deploy)
+
+Most deployment-in-progress logs are now behind a [debug](https://github.com/debug-js/debug) flag, which you can enable with a `DEBUG=mud:*` environment variable.

.changeset/wicked-pens-promise.md

@@ -0,0 +1,5 @@
+---
+"@latticexyz/world": patch
+---
+
+With [resource types in resource IDs](https://github.com/latticexyz/mud/pull/1544), the World config no longer requires table and system names to be unique.

e2e/packages/contracts/worlds.json

@@ -1,5 +1,5 @@
 {
   "31337": {
-    "address": "0x0355B7B8cb128fA5692729Ab3AAa199C1753f726"
+    "address": "0x97e55ad21ee5456964460c5465eac35861d2e797"
   }
 }

e2e/packages/sync-test/setup/deployContracts.ts

@@ -2,7 +2,7 @@ import chalk from "chalk";
 import { execa } from "execa";
 
 export function deployContracts(rpc: string) {
-  const deploymentProcess = execa("pnpm", ["mud", "deploy", "--rpc", rpc, "--disableTxWait"], {
+  const deploymentProcess = execa("pnpm", ["mud", "deploy", "--rpc", rpc], {
     cwd: "../contracts",
     stdio: "pipe",
   });

e2e/packages/test-data/generate-test-data.ts

@@ -31,14 +31,13 @@ await anvil.start();
 const rpc = `http://${anvil.host}:${anvil.port}`;
 
 console.log("deploying world");
-const { stdout, stderr } = await execa(
-  "pnpm",
-  ["mud", "deploy", "--rpc", rpc, "--disableTxWait", "--saveDeployment", "false"],
-  {
-    cwd: "../contracts",
-    stdio: "pipe",
-  }
-);
+const { stdout, stderr } = await execa("pnpm", ["mud", "deploy", "--rpc", rpc, "--saveDeployment", "false"], {
+  cwd: "../contracts",
+  stdio: "pipe",
+  env: {
+    DEBUG: "mud:*",
+  },
+});
 if (stderr) console.error(stderr);
 if (stdout) console.log(stdout);
 
@@ -101,5 +100,7 @@ const logs = await publicClient.request({
 console.log("writing", logs.length, "logs to", logsFilename);
 await fs.writeFile(logsFilename, JSON.stringify(logs, null, 2));
 
-console.log("stopping anvil");
-await anvil.stop();
+// TODO: figure out why anvil doesn't stop immediately
+// console.log("stopping anvil");
+// await anvil.stop();
+process.exit(0);

examples/minimal/packages/contracts/worlds.json

@@ -4,6 +4,6 @@
     "blockNumber": 21817970
   },
   "31337": {
-    "address": "0x5FbDB2315678afecb367f032d93F642f64180aa3"
+    "address": "0x6e9474e9c83676b9a71133ff96db43e7aa0a4342"
   }
 }

packages/cli/package.json

@@ -47,6 +47,7 @@
     "@latticexyz/world-modules": "workspace:*",
     "chalk": "^5.0.1",
     "chokidar": "^3.5.3",
+    "debug": "^4.3.4",
     "dotenv": "^16.0.3",
     "ejs": "^3.1.8",
     "ethers": "^5.7.2",
@@ -55,6 +56,7 @@
     "nice-grpc-web": "^2.0.1",
     "openurl": "^1.1.1",
     "path": "^0.12.7",
+    "rxjs": "7.5.5",
     "throttle-debounce": "^5.0.0",
     "typescript": "5.1.6",
     "viem": "1.14.0",
@@ -63,6 +65,7 @@
     "zod-validation-error": "^1.3.0"
   },
   "devDependencies": {
+    "@types/debug": "^4.1.7",
     "@types/ejs": "^3.1.1",
     "@types/glob": "^7.2.0",
     "@types/node": "^18.15.11",

packages/cli/src/commands/deploy.ts

@@ -1,43 +1,20 @@
-import type { CommandModule, Options } from "yargs";
+import type { CommandModule } from "yargs";
 import { logError } from "../utils/errors";
-import { DeployOptions, deployHandler } from "../utils/deployHandler";
+import { DeployOptions, deployOptions, runDeploy } from "../runDeploy";
 
-export const yDeployOptions = {
-  configPath: { type: "string", desc: "Path to the config file" },
-  clean: { type: "boolean", desc: "Remove the build forge artifacts and cache directories before building" },
-  printConfig: { type: "boolean", desc: "Print the resolved config" },
-  profile: { type: "string", desc: "The foundry profile to use" },
-  debug: { type: "boolean", desc: "Print debug logs, like full error messages" },
-  priorityFeeMultiplier: {
-    type: "number",
-    desc: "Multiply the estimated priority fee by the provided factor",
-    default: 1,
-  },
-  saveDeployment: { type: "boolean", desc: "Save the deployment info to a file", default: true },
-  rpc: { type: "string", desc: "The RPC URL to use. Defaults to the RPC url from the local foundry.toml" },
-  worldAddress: { type: "string", desc: "Deploy to an existing World at the given address" },
-  srcDir: { type: "string", desc: "Source directory. Defaults to foundry src directory." },
-  disableTxWait: { type: "boolean", desc: "Disable waiting for transactions to be confirmed.", default: false },
-  pollInterval: {
-    type: "number",
-    desc: "Interval in miliseconds to use to poll for transaction receipts / block inclusion",
-    default: 1000,
-  },
-  skipBuild: { type: "boolean", desc: "Skip rebuilding the contracts before deploying" },
-} satisfies Record<keyof DeployOptions, Options>;
-
-const commandModule: CommandModule<DeployOptions, DeployOptions> = {
+const commandModule: CommandModule<typeof deployOptions, DeployOptions> = {
   command: "deploy",
 
   describe: "Deploy MUD contracts",
 
   builder(yargs) {
-    return yargs.options(yDeployOptions);
+    return yargs.options(deployOptions);
   },
 
-  async handler(args) {
+  async handler(opts) {
+    // Wrap in try/catch, because yargs seems to swallow errors
     try {
-      await deployHandler(args);
+      await runDeploy(opts);
     } catch (error: any) {
       logError(error);
       process.exit(1);

packages/cli/src/commands/dev-contracts.ts

@@ -1,176 +1,101 @@
-import type { CommandModule } from "yargs";
-import {
-  anvil,
-  forge,
-  getRemappings,
-  getRpcUrl,
-  getScriptDirectory,
-  getSrcDirectory,
-} from "@latticexyz/common/foundry";
+import type { CommandModule, InferredOptionTypes } from "yargs";
+import { anvil, getScriptDirectory, getSrcDirectory } from "@latticexyz/common/foundry";
 import chalk from "chalk";
 import chokidar from "chokidar";
 import { loadConfig, resolveConfigPath } from "@latticexyz/config/node";
 import { StoreConfig } from "@latticexyz/store";
-import { tablegen } from "@latticexyz/store/codegen";
 import path from "path";
-import { debounce } from "throttle-debounce";
-import { worldgenHandler } from "./worldgen";
 import { WorldConfig } from "@latticexyz/world";
 import { homedir } from "os";
 import { rmSync } from "fs";
-import { execa } from "execa";
-import { logError } from "../utils/errors";
-import { deployHandler } from "../utils/deployHandler";
-import { printMUD } from "../utils/printMUD";
+import { deployOptions, runDeploy } from "../runDeploy";
+import { BehaviorSubject, debounceTime, exhaustMap } from "rxjs";
+import { Address } from "viem";
 
-type Options = {
-  rpc?: string;
-  configPath?: string;
+const devOptions = {
+  rpc: deployOptions.rpc,
+  configPath: deployOptions.configPath,
 };
 
-const commandModule: CommandModule<Options, Options> = {
+const commandModule: CommandModule<typeof devOptions, InferredOptionTypes<typeof devOptions>> = {
   command: "dev-contracts",
 
   describe: "Start a development server for MUD contracts",
 
   builder(yargs) {
-    return yargs.options({
-      rpc: {
-        type: "string",
-        decs: "RPC endpoint of the development node. If none is provided, an anvil instance is spawned in the background on port 8545.",
-      },
-      configPath: {
-        type: "string",
-        decs: "Path to MUD config",
-      },
-    });
+    return yargs.options(devOptions);
   },
 
-  async handler(args) {
-    // Initial cleanup
-    await forge(["clean"]);
-
-    const rpc = args.rpc ?? (await getRpcUrl());
-    const configPath = args.configPath ?? (await resolveConfigPath(args.configPath));
-    const srcDirectory = await getSrcDirectory();
-    const scriptDirectory = await getScriptDirectory();
-    const remappings = await getRemappings();
+  async handler(opts) {
+    let rpc = opts.rpc;
+    const configPath = opts.configPath ?? (await resolveConfigPath(opts.configPath));
+    const srcDir = await getSrcDirectory();
+    const scriptDir = await getScriptDirectory();
     const initialConfig = (await loadConfig(configPath)) as StoreConfig & WorldConfig;
 
-    // Initial run of all codegen steps before starting anvil
-    // (so clients can wait for everything to be ready before starting)
-    await handleConfigChange(initialConfig);
-    await handleContractsChange(initialConfig);
-
     // Start an anvil instance in the background if no RPC url is provided
-    if (!args.rpc) {
+    if (!opts.rpc) {
+      // Clean anvil cache as 1s block times can fill up the disk
+      // - https://github.com/foundry-rs/foundry/issues/3623
+      // - https://github.com/foundry-rs/foundry/issues/4989
+      // - https://github.com/foundry-rs/foundry/issues/3699
+      // - https://github.com/foundry-rs/foundry/issues/3512
       console.log(chalk.gray("Cleaning devnode cache"));
       const userHomeDir = homedir();
       rmSync(path.join(userHomeDir, ".foundry", "anvil", "tmp"), { recursive: true, force: true });
 
       const anvilArgs = ["--block-time", "1", "--block-base-fee-per-gas", "0"];
       anvil(anvilArgs);
+      rpc = "http://127.0.0.1:8545";
     }
 
-    const changedSinceLastHandled = {
-      config: false,
-      contracts: false,
-    };
-
-    const changeInProgress = {
-      current: false,
-    };
-
     // Watch for changes
-    const configWatcher = chokidar.watch([configPath, srcDirectory]);
-    configWatcher.on("all", async (_, updatePath) => {
+    const lastChange$ = new BehaviorSubject<number>(Date.now());
+    chokidar.watch([configPath, srcDir, scriptDir]).on("all", async (_, updatePath) => {
       if (updatePath.includes(configPath)) {
-        changedSinceLastHandled.config = true;
-        // We trigger contract changes if the config changed here instead of
-        // listening to changes in the codegen directory to avoid an infinite loop
-        changedSinceLastHandled.contracts = true;
+        lastChange$.next(Date.now());
       }
-
-      if (updatePath.includes(srcDirectory) || updatePath.includes(scriptDirectory)) {
+      if (updatePath.includes(srcDir) || updatePath.includes(scriptDir)) {
         // Ignore changes to codegen files to avoid an infinite loop
-        if (updatePath.includes(initialConfig.codegenDirectory)) return;
-        changedSinceLastHandled.contracts = true;
+        if (!updatePath.includes(initialConfig.codegenDirectory)) {
+          lastChange$.next(Date.now());
+        }
       }
-
-      // Trigger debounced onChange
-      handleChange();
     });
 
-    const handleChange = debounce(100, async () => {
-      // Avoid handling changes multiple times in parallel
-      if (changeInProgress.current) return;
-      changeInProgress.current = true;
-
-      // Reset dirty flags
-      const { config, contracts } = changedSinceLastHandled;
-      changedSinceLastHandled.config = false;
-      changedSinceLastHandled.contracts = false;
-
-      try {
-        // Load latest config
-        const mudConfig = (await loadConfig(configPath)) as StoreConfig & WorldConfig;
-
-        // Handle changes
-        if (config) await handleConfigChange(mudConfig);
-        if (contracts) await handleContractsChange(mudConfig);
-
-        await deploy();
-      } catch (error) {
-        console.error(chalk.red("MUD dev-contracts watcher failed to deploy config or contracts changes\n"));
-        logError(error);
-      }
-
-      changeInProgress.current = false;
-      if (changedSinceLastHandled.config || changedSinceLastHandled.contracts) {
-        console.log("Detected change while handling the previous change");
-        handleChange();
-      }
-
-      printMUD();
-      console.log("MUD watching for changes...");
-    });
-
-    /** Codegen to run if config changes */
-    async function handleConfigChange(config: StoreConfig & WorldConfig) {
-      console.log(chalk.blue("mud.config.ts changed - regenerating tables and recs types"));
-      // Run tablegen to generate tables based on the config
-      const outPath = path.join(srcDirectory, config.codegenDirectory);
-      await tablegen(config, outPath, remappings);
-    }
-
-    /** Codegen to run if contracts changed */
-    async function handleContractsChange(config: StoreConfig & WorldConfig) {
-      console.log(chalk.blue("contracts changed - regenerating interfaces and contract types"));
-
-      // Run worldgen to generate interfaces based on the systems
-      await worldgenHandler({ config, clean: true, srcDir: srcDirectory });
-
-      // Build the contracts
-      await forge(["build", "--skip", "test", "script"]);
-
-      // Generate TS type definitions for ABIs
-      await execa("mud", ["abi-ts"], { stdio: "inherit" });
-    }
-
-    /** Run after codegen if either mud config or contracts changed */
-    async function deploy() {
-      console.log(chalk.blue("redeploying World"));
-      await deployHandler({
-        configPath,
-        skipBuild: true,
-        priorityFeeMultiplier: 1,
-        disableTxWait: true,
-        pollInterval: 1000,
-        saveDeployment: true,
-        srcDir: srcDirectory,
-        rpc,
-      });
-    }
+    let worldAddress: Address | undefined;
+
+    const deploys$ = lastChange$.pipe(
+      // debounce so that a large batch of file changes only triggers a deploy after it settles down, rather than the first change it sees (and then redeploying immediately after)
+      debounceTime(200),
+      exhaustMap(async (lastChange) => {
+        if (worldAddress) {
+          console.log(chalk.blue("Change detected, rebuilding and running deploy..."));
+        }
+        // TODO: handle errors
+        const deploy = await runDeploy({
+          configPath,
+          rpc,
+          clean: true,
+          skipBuild: false,
+          printConfig: false,
+          profile: undefined,
+          saveDeployment: true,
+          worldAddress,
+          srcDir,
+        });
+        worldAddress = deploy.address;
+        // if there were changes while we were deploying, trigger it again
+        if (lastChange < lastChange$.value) {
+          lastChange$.next(lastChange$.value);
+        } else {
+          console.log(chalk.gray("Watching for file changes..."));
+        }
+        return deploy;
+      })
+    );
+
+    deploys$.subscribe();
   },
 };