use wrangler r2 bulk put for R2 cache population

.changeset/beige-aliens-tickle.md

@@ -0,0 +1,5 @@
+---
+"@opennextjs/cloudflare": minor
+---
+
+use `wrangler r2 bulk put` for R2 cache population

packages/cloudflare/README.md

@@ -55,30 +55,3 @@ Deploy your application to production with the following:
   # or
   bun opennextjs-cloudflare build && bun opennextjs-cloudflare deploy
   ```
-
-### Batch Cache Population (Optional, Recommended)
-
-For improved performance with large caches, you can enable batch upload by providing R2 credentials via .env or environment variables.
-
-Create a `.env` file in your project root (automatically loaded by the CLI):
-
-```bash
-R2_ACCESS_KEY_ID=your_access_key_id
-R2_SECRET_ACCESS_KEY=your_secret_access_key
-CF_ACCOUNT_ID=your_account_id
-```
-
-You can also set the environment variables for CI builds.
-
-**Note:**
-
-You can follow documentation https://developers.cloudflare.com/r2/api/tokens/ for creating API tokens with appropriate permissions for R2 access.
-
-**Benefits:**
-
-- Significantly faster uploads for large caches using parallel transfers
-- Reduced API calls to Cloudflare
-- Automatically enabled when credentials are provided
-
-**Fallback:**
-If these environment variables are not set, the CLI will use standard Wrangler uploads. Both methods work correctly - batch upload is simply faster for large caches.

packages/cloudflare/package.json

@@ -55,11 +55,9 @@
 		"@ast-grep/napi": "0.40.0",
 		"@dotenvx/dotenvx": "catalog:",
 		"@opennextjs/aws": "3.8.5",
-		"@types/rclone.js": "^0.6.3",
 		"cloudflare": "^4.4.1",
 		"enquirer": "^2.4.1",
 		"glob": "catalog:",
-		"rclone.js": "^0.6.6",
 		"ts-tqdm": "^0.8.6",
 		"yargs": "catalog:"
 	},

packages/cloudflare/src/api/cloudflare-context.ts

@@ -79,13 +79,8 @@ declare global {
 		CF_PREVIEW_DOMAIN?: string;
 		// Should have the `Workers Scripts:Read` permission
 		CF_WORKERS_SCRIPTS_API_TOKEN?: string;
-
-		// Cloudflare account id - needed for skew protection and R2 batch population
+		// Cloudflare account id - needed for skew protection
 		CF_ACCOUNT_ID?: string;
-
-		// R2 API credentials for batch cache population (optional, enables faster uploads)
-		R2_ACCESS_KEY_ID?: string;
-		R2_SECRET_ACCESS_KEY?: string;
 	}
 }
 

packages/cloudflare/src/cli/commands/populate-cache.spec.ts

@@ -78,44 +78,7 @@ vi.mock("./helpers.js", () => ({
 	quoteShellMeta: vi.fn((s) => s),
 }));
 
-// Mock rclone.js promises API to simulate successful copy operations by default
-vi.mock("rclone.js", () => ({
-	default: {
-		promises: {
-			copy: vi.fn(() => Promise.resolve("")),
-		},
-	},
-}));
-
 describe("populateCache", () => {
-	// Test fixtures
-	const createTestBuildOptions = (): BuildOptions =>
-		({
-			outputDir: "/test/output",
-		}) as BuildOptions;
-
-	const createTestOpenNextConfig = () => ({
-		default: {
-			override: {
-				incrementalCache: "cf-r2-incremental-cache",
-			},
-		},
-	});
-
-	const createTestWranglerConfig = () => ({
-		r2_buckets: [
-			{
-				binding: "NEXT_INC_CACHE_R2_BUCKET",
-				bucket_name: "test-bucket",
-			},
-		],
-	});
-
-	const createTestPopulateCacheOptions = () => ({
-		target: "local" as const,
-		shouldUsePreviewId: false,
-	});
-
 	const setupMockFileSystem = () => {
 		mockFs({
 			"/test/output": {
@@ -132,153 +95,48 @@ describe("populateCache", () => {
 		});
 	};
 
-	describe("R2 incremental cache", () => {
-		afterEach(() => {
-			mockFs.restore();
-			vi.unstubAllEnvs();
-		});
-
-		test("uses sequential upload for local target (skips batch upload)", async () => {
-			const { runWrangler } = await import("../utils/run-wrangler.js");
-			const rcloneModule = (await import("rclone.js")).default;
-
-			setupMockFileSystem();
-			vi.mocked(runWrangler).mockClear();
-			vi.mocked(rcloneModule.promises.copy).mockClear();
-
-			// Test with local target - should skip batch upload even with credentials
-			await populateCache(
-				createTestBuildOptions(),
-				createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
-				createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
-				{ target: "local" as const, shouldUsePreviewId: false },
-				{
-					R2_ACCESS_KEY_ID: "test_access_key",
-					R2_SECRET_ACCESS_KEY: "test_secret_key",
-					CF_ACCOUNT_ID: "test_account_id",
-				} as any // eslint-disable-line @typescript-eslint/no-explicit-any
-			);
-
-			// Should use sequential upload (runWrangler), not batch upload (rclone.js)
-			expect(runWrangler).toHaveBeenCalled();
-			expect(rcloneModule.promises.copy).not.toHaveBeenCalled();
-		});
-
-		test("uses sequential upload when R2 credentials are not provided", async () => {
-			const { runWrangler } = await import("../utils/run-wrangler.js");
-			const rcloneModule = (await import("rclone.js")).default;
-
-			setupMockFileSystem();
-			vi.mocked(runWrangler).mockClear();
-			vi.mocked(rcloneModule.promises.copy).mockClear();
-
-			// Test uses partial types for simplicity - full config not needed
-			// Pass empty envVars to simulate no R2 credentials
-			await populateCache(
-				createTestBuildOptions(),
-				createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
-				createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
-				createTestPopulateCacheOptions(),
-				{} as any // eslint-disable-line @typescript-eslint/no-explicit-any
-			);
-
-			expect(runWrangler).toHaveBeenCalled();
-			expect(rcloneModule.promises.copy).not.toHaveBeenCalled();
-		});
-
-		test("uses batch upload with temporary config for remote target when R2 credentials are provided", async () => {
-			const rcloneModule = (await import("rclone.js")).default;
-
-			setupMockFileSystem();
-			vi.mocked(rcloneModule.promises.copy).mockClear();
-
-			// Test uses partial types for simplicity - full config not needed
-			// Pass envVars with R2 credentials and remote target to enable batch upload
-			await populateCache(
-				createTestBuildOptions(),
-				createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
-				createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
-				{ target: "remote" as const, shouldUsePreviewId: false },
-				{
-					R2_ACCESS_KEY_ID: "test_access_key",
-					R2_SECRET_ACCESS_KEY: "test_secret_key",
-					CF_ACCOUNT_ID: "test_account_id",
-				} as any // eslint-disable-line @typescript-eslint/no-explicit-any
-			);
-
-			// Verify batch upload was used with correct parameters and temporary config
-			expect(rcloneModule.promises.copy).toHaveBeenCalledWith(
-				expect.any(String), // staging directory
-				"r2:test-bucket",
-				expect.objectContaining({
-					progress: true,
-					transfers: 16,
-					checkers: 8,
-					env: expect.objectContaining({
-						RCLONE_CONFIG: expect.stringMatching(/rclone-config-\d+\.conf$/),
-					}),
-				})
-			);
-		});
-
-		test("handles rclone errors with status > 0 for remote target", async () => {
-			const { runWrangler } = await import("../utils/run-wrangler.js");
-			const rcloneModule = (await import("rclone.js")).default;
-
-			setupMockFileSystem();
-
-			// Mock rclone failure - Promise rejection
-			vi.mocked(rcloneModule.promises.copy).mockRejectedValueOnce(
-				new Error("rclone copy failed with exit code 7")
-			);
-
-			vi.mocked(runWrangler).mockClear();
-
-			// Pass envVars with R2 credentials and remote target to enable batch upload (which will fail)
-			await populateCache(
-				createTestBuildOptions(),
-				createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
-				createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
-				{ target: "remote" as const, shouldUsePreviewId: false },
-				{
-					R2_ACCESS_KEY_ID: "test_access_key",
-					R2_SECRET_ACCESS_KEY: "test_secret_key",
-					CF_ACCOUNT_ID: "test_account_id",
-				} as any // eslint-disable-line @typescript-eslint/no-explicit-any
-			);
-
-			// Should fall back to sequential upload when batch upload fails
-			expect(runWrangler).toHaveBeenCalled();
-		});
-
-		test("handles rclone errors with stderr output for remote target", async () => {
-			const { runWrangler } = await import("../utils/run-wrangler.js");
-			const rcloneModule = (await import("rclone.js")).default;
-
-			setupMockFileSystem();
-
-			// Mock rclone error - Promise rejection with stderr message
-			vi.mocked(rcloneModule.promises.copy).mockRejectedValueOnce(
-				new Error("ERROR : Failed to copy: AccessDenied: Access Denied (403)")
-			);
-
-			vi.mocked(runWrangler).mockClear();
-
-			// Pass envVars with R2 credentials and remote target to enable batch upload (which will fail)
-			await populateCache(
-				createTestBuildOptions(),
-				createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
-				createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
-				{ target: "remote" as const, shouldUsePreviewId: false },
-				{
-					R2_ACCESS_KEY_ID: "test_access_key",
-					R2_SECRET_ACCESS_KEY: "test_secret_key",
-					CF_ACCOUNT_ID: "test_account_id",
-				} as any // eslint-disable-line @typescript-eslint/no-explicit-any
-			);
-
-			// Should fall back to standard upload when batch upload fails
-			expect(runWrangler).toHaveBeenCalled();
-		});
-	});
+	describe.each([{ target: "local" as const }, { target: "remote" as const }])(
+		"R2 incremental cache",
+		({ target }) => {
+			afterEach(() => {
+				mockFs.restore();
+			});
+
+			test(target, async () => {
+				const { runWrangler } = await import("../utils/run-wrangler.js");
+
+				setupMockFileSystem();
+				vi.mocked(runWrangler).mockClear();
+
+				await populateCache(
+					{
+						outputDir: "/test/output",
+					} as BuildOptions,
+					{
+						default: {
+							override: {
+								incrementalCache: "cf-r2-incremental-cache",
+							},
+						},
+					} as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+					{
+						r2_buckets: [
+							{
+								binding: "NEXT_INC_CACHE_R2_BUCKET",
+								bucket_name: "test-bucket",
+							},
+						],
+					} as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+					{ target, shouldUsePreviewId: false },
+					{} as any // eslint-disable-line @typescript-eslint/no-explicit-any
+				);
+
+				expect(runWrangler).toHaveBeenCalledWith(
+					expect.anything(),
+					expect.arrayContaining(["r2 bulk put", "test-bucket"]),
+					expect.objectContaining({ target })
+				);
+			});
+		}
+	);
 });