feat: auto-populating d1 cache data

.changeset/tame-icons-shave.md

@@ -0,0 +1,5 @@
+---
+"@opennextjs/cloudflare": patch
+---
+
+feat: auto-populating d1 cache data

examples/e2e/app-router/package.json

@@ -10,8 +10,7 @@
     "lint": "next lint",
     "clean": "rm -rf .turbo node_modules .next .open-next",
     "d1:clean": "wrangler d1 execute NEXT_CACHE_D1 --command \"DROP TABLE IF EXISTS tags; DROP TABLE IF EXISTS revalidations\"",
-    "d1:setup": "wrangler d1 execute NEXT_CACHE_D1 --file .open-next/cloudflare/cache-assets-manifest.sql",
-    "build:worker": "pnpm opennextjs-cloudflare && pnpm d1:clean && pnpm d1:setup",
+    "build:worker": "pnpm d1:clean && pnpm opennextjs-cloudflare --populateCache=local",
     "preview": "pnpm build:worker && pnpm wrangler dev",
     "e2e": "playwright test -c e2e/playwright.config.ts"
   },

packages/cloudflare/src/cli/args.ts

@@ -2,48 +2,70 @@ import { mkdirSync, type Stats, statSync } from "node:fs";
 import { resolve } from "node:path";
 import { parseArgs } from "node:util";
 
+import type { CacheBindingMode } from "./build/utils/index.js";
+import { isCacheBindingMode } from "./build/utils/index.js";
+
 export function getArgs(): {
   skipNextBuild: boolean;
   skipWranglerConfigCheck: boolean;
   outputDir?: string;
   minify: boolean;
+  populateCache?: { mode: CacheBindingMode; onlyPopulateWithoutBuilding: boolean };
 } {
-  const { skipBuild, skipWranglerConfigCheck, output, noMinify } = parseArgs({
-    options: {
-      skipBuild: {
-        type: "boolean",
-        short: "s",
-        default: false,
-      },
-      output: {
-        type: "string",
-        short: "o",
-      },
-      noMinify: {
-        type: "boolean",
-        default: false,
+  const { skipBuild, skipWranglerConfigCheck, output, noMinify, populateCache, onlyPopulateCache } =
+    parseArgs({
+      options: {
+        skipBuild: {
+          type: "boolean",
+          short: "s",
+          default: false,
+        },
+        output: {
+          type: "string",
+          short: "o",
+        },
+        noMinify: {
+          type: "boolean",
+          default: false,
+        },
+        skipWranglerConfigCheck: {
+          type: "boolean",
+          default: false,
+        },
+        populateCache: {
+          type: "string",
+        },
+        onlyPopulateCache: {
+          type: "boolean",
+          default: false,
+        },
       },
-      skipWranglerConfigCheck: {
-        type: "boolean",
-        default: false,
-      },
-    },
-    allowPositionals: false,
-  }).values;
+      allowPositionals: false,
+    }).values;
 
   const outputDir = output ? resolve(output) : undefined;
 
   if (outputDir) {
     assertDirArg(outputDir, "output", true);
   }
 
+  if (
+    (populateCache !== undefined || onlyPopulateCache) &&
+    (!populateCache?.length || !isCacheBindingMode(populateCache))
+  ) {
+    throw new Error(`Error: missing mode for populate cache flag, expected 'local' | 'remote'`);
+  }
+
   return {
     outputDir,
     skipNextBuild: skipBuild || ["1", "true", "yes"].includes(String(process.env.SKIP_NEXT_APP_BUILD)),
     skipWranglerConfigCheck:
       skipWranglerConfigCheck ||
       ["1", "true", "yes"].includes(String(process.env.SKIP_WRANGLER_CONFIG_CHECK)),
     minify: !noMinify,
+    populateCache: populateCache
+      ? { mode: populateCache, onlyPopulateWithoutBuilding: !!onlyPopulateCache }
+      : undefined,
   };
 }
 

packages/cloudflare/src/cli/build/build.ts

@@ -20,6 +20,7 @@ import {
   createOpenNextConfigIfNotExistent,
   createWranglerConfigIfNotExistent,
   ensureCloudflareConfig,
+  populateCache,
 } from "./utils/index.js";
 import { getVersion } from "./utils/version.js";
 
@@ -62,6 +63,11 @@ export async function build(projectOpts: ProjectOptions): Promise<void> {
   logger.info(`@opennextjs/cloudflare version: ${cloudflare}`);
   logger.info(`@opennextjs/aws version: ${aws}`);
 
+  if (projectOpts.populateCache?.onlyPopulateWithoutBuilding) {
+    populateCache(options, config, projectOpts.populateCache.mode);
+    return;
+  }
+
   if (projectOpts.skipNextBuild) {
     logger.warn("Skipping Next.js build");
   } else {
@@ -103,6 +109,10 @@ export async function build(projectOpts: ProjectOptions): Promise<void> {
     await createWranglerConfigIfNotExistent(projectOpts);
   }
 
+  if (projectOpts.populateCache) {
+    populateCache(options, config, projectOpts.populateCache.mode);
+  }
+
   logger.info("OpenNext build complete.");
 }
 

packages/cloudflare/src/cli/build/utils/index.ts

@@ -4,3 +4,4 @@ export * from "./ensure-cf-config.js";
 export * from "./extract-project-env-vars.js";
 export * from "./needs-experimental-react.js";
 export * from "./normalize-path.js";
+export * from "./populate-cache.js";

packages/cloudflare/src/cli/build/utils/populate-cache.ts

@@ -0,0 +1,78 @@
+import { spawnSync } from "node:child_process";
+import { existsSync } from "node:fs";
+import path from "node:path";
+
+import type { BuildOptions } from "@opennextjs/aws/build/helper.js";
+import logger from "@opennextjs/aws/logger.js";
+import type {
+  IncludedIncrementalCache,
+  IncludedTagCache,
+  LazyLoadedOverride,
+  OpenNextConfig,
+} from "@opennextjs/aws/types/open-next.js";
+import type { IncrementalCache, TagCache } from "@opennextjs/aws/types/overrides.js";
+
+export type CacheBindingMode = "local" | "remote";
+
+async function resolveCacheName(
+  value:
+    | IncludedIncrementalCache
+    | IncludedTagCache
+    | LazyLoadedOverride<IncrementalCache>
+    | LazyLoadedOverride<TagCache>
+) {
+  return typeof value === "function" ? (await value()).name : value;
+}
+
+function runWrangler(opts: BuildOptions, mode: CacheBindingMode, args: string[]) {
+  const result = spawnSync(
+    opts.packager,
+    ["exec", "wrangler", ...args, mode === "remote" && "--remote"].filter((v): v is string => !!v),
+    {
+      shell: true,
+      stdio: ["ignore", "ignore", "inherit"],
+    }
+  );
+
+  if (result.status !== 0) {
+    logger.error("Failed to populate cache");
+    process.exit(1);
+  } else {
+    logger.info("Successfully populated cache");
+  }
+}
+
+export async function populateCache(opts: BuildOptions, config: OpenNextConfig, mode: CacheBindingMode) {
+  const { incrementalCache, tagCache } = config.default.override ?? {};
+
+  if (!existsSync(opts.outputDir)) {
+    logger.error("Unable to populate cache: Open Next build not found");
+    process.exit(1);
+  }
+
+  if (!config.dangerous?.disableIncrementalCache && incrementalCache) {
+    logger.info("Incremental cache does not need populating");
+  }
+
+  if (!config.dangerous?.disableTagCache && !config.dangerous?.disableIncrementalCache && tagCache) {
+    const name = await resolveCacheName(tagCache);
+    switch (name) {
+      case "d1-tag-cache": {
+        logger.info("\nPopulating D1 tag cache...");
+
+        runWrangler(opts, mode, [
+          "d1 execute",
+          "NEXT_CACHE_D1",
+          `--file ${JSON.stringify(path.join(opts.outputDir, "cloudflare/cache-assets-manifest.sql"))}`,
+        ]);
+        break;
+      }
+      default:
+        logger.info("Tag cache does not need populating");
+    }
+  }
+}
+
+export function isCacheBindingMode(v: string | undefined): v is CacheBindingMode {
+  return !!v && ["local", "remote"].includes(v);
+}

packages/cloudflare/src/cli/index.ts

@@ -6,12 +6,13 @@ import { build } from "./build/build.js";
 
 const nextAppDir = process.cwd();
 
-const { skipNextBuild, skipWranglerConfigCheck, outputDir, minify } = getArgs();
+const { skipNextBuild, skipWranglerConfigCheck, outputDir, minify, populateCache } = getArgs();
 
 await build({
   sourceDir: nextAppDir,
   outputDir: resolve(outputDir ?? nextAppDir, ".open-next"),
   skipNextBuild,
   skipWranglerConfigCheck,
   minify,
+  populateCache,
 });

packages/cloudflare/src/cli/project-options.ts

@@ -1,3 +1,5 @@
+import type { CacheBindingMode } from "./build/utils/index.js";
+
 export type ProjectOptions = {
   // Next app root folder
   sourceDir: string;
@@ -9,4 +11,5 @@ export type ProjectOptions = {
   skipWranglerConfigCheck: boolean;
   // Whether minification of the worker should be enabled
   minify: boolean;
+  populateCache?: { mode: CacheBindingMode; onlyPopulateWithoutBuilding: boolean };
 };