From 81aae0e57e6caffc369381f9d28de2f6535464e3 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 3 Mar 2025 22:17:29 +0000 Subject: [PATCH 1/5] feat: auto-populating d1 cache data --- .changeset/tame-icons-shave.md | 5 ++ examples/e2e/app-router/package.json | 3 +- packages/cloudflare/src/cli/args.ts | 63 ++++++++++++------- packages/cloudflare/src/cli/build/build.ts | 10 +++ .../cloudflare/src/cli/build/utils/index.ts | 1 + .../src/cli/build/utils/populate-cache.ts | 61 ++++++++++++++++++ packages/cloudflare/src/cli/index.ts | 3 +- .../cloudflare/src/cli/project-options.ts | 3 + 8 files changed, 125 insertions(+), 24 deletions(-) create mode 100644 .changeset/tame-icons-shave.md create mode 100644 packages/cloudflare/src/cli/build/utils/populate-cache.ts diff --git a/.changeset/tame-icons-shave.md b/.changeset/tame-icons-shave.md new file mode 100644 index 000000000..8dfbe7871 --- /dev/null +++ b/.changeset/tame-icons-shave.md @@ -0,0 +1,5 @@ +--- +"@opennextjs/cloudflare": patch +--- + +feat: auto-populating d1 cache data diff --git a/examples/e2e/app-router/package.json b/examples/e2e/app-router/package.json index 3d1037d12..9d26a4c6e 100644 --- a/examples/e2e/app-router/package.json +++ b/examples/e2e/app-router/package.json @@ -10,8 +10,7 @@ "lint": "next lint", "clean": "rm -rf .turbo node_modules .next .open-next", "d1:clean": "wrangler d1 execute NEXT_CACHE_D1 --command \"DROP TABLE IF EXISTS tags; DROP TABLE IF EXISTS revalidations\"", - "d1:setup": "wrangler d1 execute NEXT_CACHE_D1 --file .open-next/cloudflare/cache-assets-manifest.sql", - "build:worker": "pnpm opennextjs-cloudflare && pnpm d1:clean && pnpm d1:setup", + "build:worker": "pnpm d1:clean && pnpm opennextjs-cloudflare --populateCache=local", "preview": "pnpm build:worker && pnpm wrangler dev", "e2e": "playwright test -c e2e/playwright.config.ts" }, diff --git a/packages/cloudflare/src/cli/args.ts b/packages/cloudflare/src/cli/args.ts index f64d0d731..b29ce67c2 100644 --- a/packages/cloudflare/src/cli/args.ts +++ b/packages/cloudflare/src/cli/args.ts @@ -2,34 +2,45 @@ import { mkdirSync, type Stats, statSync } from "node:fs"; import { resolve } from "node:path"; import { parseArgs } from "node:util"; +import { CacheBindingMode } from "./build/utils/index.js"; + export function getArgs(): { skipNextBuild: boolean; skipWranglerConfigCheck: boolean; outputDir?: string; minify: boolean; + populateCache?: { mode: "local" | "remote"; onlyPopulate: boolean }; } { - const { skipBuild, skipWranglerConfigCheck, output, noMinify } = parseArgs({ - options: { - skipBuild: { - type: "boolean", - short: "s", - default: false, - }, - output: { - type: "string", - short: "o", - }, - noMinify: { - type: "boolean", - default: false, + const { skipBuild, skipWranglerConfigCheck, output, noMinify, populateCache, onlyPopulateCache } = + parseArgs({ + options: { + skipBuild: { + type: "boolean", + short: "s", + default: false, + }, + output: { + type: "string", + short: "o", + }, + noMinify: { + type: "boolean", + default: false, + }, + skipWranglerConfigCheck: { + type: "boolean", + default: false, + }, + populateCache: { + type: "string", + }, + onlyPopulateCache: { + type: "boolean", + default: false, + }, }, - skipWranglerConfigCheck: { - type: "boolean", - default: false, - }, - }, - allowPositionals: false, - }).values; + allowPositionals: false, + }).values; const outputDir = output ? resolve(output) : undefined; @@ -37,6 +48,13 @@ export function getArgs(): { assertDirArg(outputDir, "output", true); } + if ( + (populateCache !== undefined || onlyPopulateCache) && + (!populateCache?.length || !["local", "remote"].includes(populateCache)) + ) { + throw new Error(`Error: missing mode for populate cache flag, expected 'local' | 'remote'`); + } + return { outputDir, skipNextBuild: skipBuild || ["1", "true", "yes"].includes(String(process.env.SKIP_NEXT_APP_BUILD)), @@ -44,6 +62,9 @@ export function getArgs(): { skipWranglerConfigCheck || ["1", "true", "yes"].includes(String(process.env.SKIP_WRANGLER_CONFIG_CHECK)), minify: !noMinify, + populateCache: populateCache + ? { mode: populateCache as CacheBindingMode, onlyPopulate: !!onlyPopulateCache } + : undefined, }; } diff --git a/packages/cloudflare/src/cli/build/build.ts b/packages/cloudflare/src/cli/build/build.ts index 110f563b8..4148b92be 100644 --- a/packages/cloudflare/src/cli/build/build.ts +++ b/packages/cloudflare/src/cli/build/build.ts @@ -20,6 +20,7 @@ import { createOpenNextConfigIfNotExistent, createWranglerConfigIfNotExistent, ensureCloudflareConfig, + populateCache, } from "./utils/index.js"; import { getVersion } from "./utils/version.js"; @@ -62,6 +63,11 @@ export async function build(projectOpts: ProjectOptions): Promise { logger.info(`@opennextjs/cloudflare version: ${cloudflare}`); logger.info(`@opennextjs/aws version: ${aws}`); + if (projectOpts.populateCache?.onlyPopulate) { + populateCache(options, config, projectOpts.populateCache.mode); + return; + } + if (projectOpts.skipNextBuild) { logger.warn("Skipping Next.js build"); } else { @@ -103,6 +109,10 @@ export async function build(projectOpts: ProjectOptions): Promise { await createWranglerConfigIfNotExistent(projectOpts); } + if (projectOpts.populateCache) { + populateCache(options, config, projectOpts.populateCache.mode); + } + logger.info("OpenNext build complete."); } diff --git a/packages/cloudflare/src/cli/build/utils/index.ts b/packages/cloudflare/src/cli/build/utils/index.ts index cca97f023..e7fb383b6 100644 --- a/packages/cloudflare/src/cli/build/utils/index.ts +++ b/packages/cloudflare/src/cli/build/utils/index.ts @@ -4,3 +4,4 @@ export * from "./ensure-cf-config.js"; export * from "./extract-project-env-vars.js"; export * from "./needs-experimental-react.js"; export * from "./normalize-path.js"; +export * from "./populate-cache.js"; diff --git a/packages/cloudflare/src/cli/build/utils/populate-cache.ts b/packages/cloudflare/src/cli/build/utils/populate-cache.ts new file mode 100644 index 000000000..c9aa07335 --- /dev/null +++ b/packages/cloudflare/src/cli/build/utils/populate-cache.ts @@ -0,0 +1,61 @@ +import { spawnSync } from "node:child_process"; +import path from "node:path"; + +import type { BuildOptions } from "@opennextjs/aws/build/helper.js"; +import logger from "@opennextjs/aws/logger.js"; +import type { + IncludedIncrementalCache, + IncludedTagCache, + LazyLoadedOverride, + OpenNextConfig, +} from "@opennextjs/aws/types/open-next.js"; +import type { IncrementalCache, TagCache } from "@opennextjs/aws/types/overrides.js"; + +export type CacheBindingMode = "local" | "remote"; + +async function resolveCacheName( + value: + | IncludedIncrementalCache + | IncludedTagCache + | LazyLoadedOverride + | LazyLoadedOverride +) { + return typeof value === "function" ? (await value()).name : value; +} + +function runWrangler(opts: BuildOptions, mode: CacheBindingMode, args: string[]) { + const result = spawnSync( + opts.packager, + ["exec", "wrangler", ...args, mode === "remote" && "--remote"].filter((v): v is string => !!v), + { + shell: true, + stdio: ["ignore", "ignore", "inherit"], + } + ); + + if (result.status !== 0) { + logger.error("Failed to populate cache"); + } else { + logger.info("Successfully populated cache"); + } +} + +export async function populateCache(opts: BuildOptions, config: OpenNextConfig, mode: CacheBindingMode) { + const { tagCache } = config.default.override ?? {}; + + if (tagCache) { + const name = await resolveCacheName(tagCache); + switch (name) { + case "d1-tag-cache": { + logger.info("\nPopulating D1 tag cache..."); + + runWrangler(opts, mode, [ + "d1 execute", + "NEXT_CACHE_D1", + `--file ${JSON.stringify(path.join(opts.outputDir, "cloudflare/cache-assets-manifest.sql"))}`, + ]); + break; + } + } + } +} diff --git a/packages/cloudflare/src/cli/index.ts b/packages/cloudflare/src/cli/index.ts index 2a6c654e4..be3c1e763 100644 --- a/packages/cloudflare/src/cli/index.ts +++ b/packages/cloudflare/src/cli/index.ts @@ -6,7 +6,7 @@ import { build } from "./build/build.js"; const nextAppDir = process.cwd(); -const { skipNextBuild, skipWranglerConfigCheck, outputDir, minify } = getArgs(); +const { skipNextBuild, skipWranglerConfigCheck, outputDir, minify, populateCache } = getArgs(); await build({ sourceDir: nextAppDir, @@ -14,4 +14,5 @@ await build({ skipNextBuild, skipWranglerConfigCheck, minify, + populateCache, }); diff --git a/packages/cloudflare/src/cli/project-options.ts b/packages/cloudflare/src/cli/project-options.ts index 0ceb3d96c..f83beef4d 100644 --- a/packages/cloudflare/src/cli/project-options.ts +++ b/packages/cloudflare/src/cli/project-options.ts @@ -1,3 +1,5 @@ +import type { CacheBindingMode } from "./build/utils/index.js"; + export type ProjectOptions = { // Next app root folder sourceDir: string; @@ -9,4 +11,5 @@ export type ProjectOptions = { skipWranglerConfigCheck: boolean; // Whether minification of the worker should be enabled minify: boolean; + populateCache?: { mode: CacheBindingMode; onlyPopulate: boolean }; }; From c19f43fea25cf1128f2575ab942ae6e439259672 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 4 Mar 2025 08:57:00 +0000 Subject: [PATCH 2/5] checks for output directory / enabled --- .../src/cli/build/utils/populate-cache.ts | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/packages/cloudflare/src/cli/build/utils/populate-cache.ts b/packages/cloudflare/src/cli/build/utils/populate-cache.ts index c9aa07335..27f5c4ae9 100644 --- a/packages/cloudflare/src/cli/build/utils/populate-cache.ts +++ b/packages/cloudflare/src/cli/build/utils/populate-cache.ts @@ -1,4 +1,5 @@ import { spawnSync } from "node:child_process"; +import { existsSync } from "node:fs"; import path from "node:path"; import type { BuildOptions } from "@opennextjs/aws/build/helper.js"; @@ -35,15 +36,25 @@ function runWrangler(opts: BuildOptions, mode: CacheBindingMode, args: string[]) if (result.status !== 0) { logger.error("Failed to populate cache"); + process.exit(1); } else { logger.info("Successfully populated cache"); } } export async function populateCache(opts: BuildOptions, config: OpenNextConfig, mode: CacheBindingMode) { - const { tagCache } = config.default.override ?? {}; + const { incrementalCache, tagCache } = config.default.override ?? {}; - if (tagCache) { + if (!existsSync(opts.outputDir)) { + logger.error("Unable to populate cache: Open Next build not found"); + process.exit(1); + } + + if (!config.dangerous?.disableIncrementalCache && incrementalCache) { + logger.info("Incremental cache does not need populating"); + } + + if (!config.dangerous?.disableTagCache && tagCache) { const name = await resolveCacheName(tagCache); switch (name) { case "d1-tag-cache": { @@ -56,6 +67,8 @@ export async function populateCache(opts: BuildOptions, config: OpenNextConfig, ]); break; } + default: + logger.info("Tag cache does not need populating"); } } } From 697e2f8e0434c0f6c250d9b201c83bd27616988d Mon Sep 17 00:00:00 2001 From: James Anderson Date: Tue, 4 Mar 2025 09:50:26 +0000 Subject: [PATCH 3/5] Update packages/cloudflare/src/cli/build/utils/populate-cache.ts Co-authored-by: conico974 --- packages/cloudflare/src/cli/build/utils/populate-cache.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cloudflare/src/cli/build/utils/populate-cache.ts b/packages/cloudflare/src/cli/build/utils/populate-cache.ts index 27f5c4ae9..f5533289d 100644 --- a/packages/cloudflare/src/cli/build/utils/populate-cache.ts +++ b/packages/cloudflare/src/cli/build/utils/populate-cache.ts @@ -54,7 +54,7 @@ export async function populateCache(opts: BuildOptions, config: OpenNextConfig, logger.info("Incremental cache does not need populating"); } - if (!config.dangerous?.disableTagCache && tagCache) { + if (!config.dangerous?.disableTagCache && !config.dangerous?.disableIncrementalCache && tagCache) { const name = await resolveCacheName(tagCache); switch (name) { case "d1-tag-cache": { From 87b39b0e3fbfe53c64094a2bdf312f0f939b2d7e Mon Sep 17 00:00:00 2001 From: James Date: Sat, 8 Mar 2025 20:46:04 +0000 Subject: [PATCH 4/5] add suggestions --- packages/cloudflare/src/cli/args.ts | 11 +++++------ .../cloudflare/src/cli/build/utils/populate-cache.ts | 4 ++++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/packages/cloudflare/src/cli/args.ts b/packages/cloudflare/src/cli/args.ts index b29ce67c2..f21907694 100644 --- a/packages/cloudflare/src/cli/args.ts +++ b/packages/cloudflare/src/cli/args.ts @@ -2,14 +2,15 @@ import { mkdirSync, type Stats, statSync } from "node:fs"; import { resolve } from "node:path"; import { parseArgs } from "node:util"; -import { CacheBindingMode } from "./build/utils/index.js"; +import type { CacheBindingMode } from "./build/utils/index.js"; +import { isCacheBindingMode } from "./build/utils/index.js"; export function getArgs(): { skipNextBuild: boolean; skipWranglerConfigCheck: boolean; outputDir?: string; minify: boolean; - populateCache?: { mode: "local" | "remote"; onlyPopulate: boolean }; + populateCache?: { mode: CacheBindingMode; onlyPopulate: boolean }; } { const { skipBuild, skipWranglerConfigCheck, output, noMinify, populateCache, onlyPopulateCache } = parseArgs({ @@ -50,7 +51,7 @@ export function getArgs(): { if ( (populateCache !== undefined || onlyPopulateCache) && - (!populateCache?.length || !["local", "remote"].includes(populateCache)) + (!populateCache?.length || !isCacheBindingMode(populateCache)) ) { throw new Error(`Error: missing mode for populate cache flag, expected 'local' | 'remote'`); } @@ -62,9 +63,7 @@ export function getArgs(): { skipWranglerConfigCheck || ["1", "true", "yes"].includes(String(process.env.SKIP_WRANGLER_CONFIG_CHECK)), minify: !noMinify, - populateCache: populateCache - ? { mode: populateCache as CacheBindingMode, onlyPopulate: !!onlyPopulateCache } - : undefined, + populateCache: populateCache ? { mode: populateCache, onlyPopulate: !!onlyPopulateCache } : undefined, }; } diff --git a/packages/cloudflare/src/cli/build/utils/populate-cache.ts b/packages/cloudflare/src/cli/build/utils/populate-cache.ts index f5533289d..ad38ede79 100644 --- a/packages/cloudflare/src/cli/build/utils/populate-cache.ts +++ b/packages/cloudflare/src/cli/build/utils/populate-cache.ts @@ -72,3 +72,7 @@ export async function populateCache(opts: BuildOptions, config: OpenNextConfig, } } } + +export function isCacheBindingMode(v: string | undefined): v is CacheBindingMode { + return !!v && ["local", "remote"].includes(v); +} From 76e2c5d92e8ab43c8ef83e8833258883d528ff46 Mon Sep 17 00:00:00 2001 From: James Date: Sat, 8 Mar 2025 20:49:16 +0000 Subject: [PATCH 5/5] rename onlyPopulate to onlyPopulateWithoutBuilding --- packages/cloudflare/src/cli/args.ts | 6 ++++-- packages/cloudflare/src/cli/build/build.ts | 2 +- packages/cloudflare/src/cli/project-options.ts | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/cloudflare/src/cli/args.ts b/packages/cloudflare/src/cli/args.ts index f21907694..5a546ad99 100644 --- a/packages/cloudflare/src/cli/args.ts +++ b/packages/cloudflare/src/cli/args.ts @@ -10,7 +10,7 @@ export function getArgs(): { skipWranglerConfigCheck: boolean; outputDir?: string; minify: boolean; - populateCache?: { mode: CacheBindingMode; onlyPopulate: boolean }; + populateCache?: { mode: CacheBindingMode; onlyPopulateWithoutBuilding: boolean }; } { const { skipBuild, skipWranglerConfigCheck, output, noMinify, populateCache, onlyPopulateCache } = parseArgs({ @@ -63,7 +63,9 @@ export function getArgs(): { skipWranglerConfigCheck || ["1", "true", "yes"].includes(String(process.env.SKIP_WRANGLER_CONFIG_CHECK)), minify: !noMinify, - populateCache: populateCache ? { mode: populateCache, onlyPopulate: !!onlyPopulateCache } : undefined, + populateCache: populateCache + ? { mode: populateCache, onlyPopulateWithoutBuilding: !!onlyPopulateCache } + : undefined, }; } diff --git a/packages/cloudflare/src/cli/build/build.ts b/packages/cloudflare/src/cli/build/build.ts index 4148b92be..c2b8de995 100644 --- a/packages/cloudflare/src/cli/build/build.ts +++ b/packages/cloudflare/src/cli/build/build.ts @@ -63,7 +63,7 @@ export async function build(projectOpts: ProjectOptions): Promise { logger.info(`@opennextjs/cloudflare version: ${cloudflare}`); logger.info(`@opennextjs/aws version: ${aws}`); - if (projectOpts.populateCache?.onlyPopulate) { + if (projectOpts.populateCache?.onlyPopulateWithoutBuilding) { populateCache(options, config, projectOpts.populateCache.mode); return; } diff --git a/packages/cloudflare/src/cli/project-options.ts b/packages/cloudflare/src/cli/project-options.ts index f83beef4d..15b23259d 100644 --- a/packages/cloudflare/src/cli/project-options.ts +++ b/packages/cloudflare/src/cli/project-options.ts @@ -11,5 +11,5 @@ export type ProjectOptions = { skipWranglerConfigCheck: boolean; // Whether minification of the worker should be enabled minify: boolean; - populateCache?: { mode: CacheBindingMode; onlyPopulate: boolean }; + populateCache?: { mode: CacheBindingMode; onlyPopulateWithoutBuilding: boolean }; };