feat: use remote dev for r2 cache population

.changeset/fix-r2-cache-upload.md

@@ -0,0 +1,7 @@
+---
+"@opennextjs/cloudflare": minor
+---
+
+Use remote dev for R2 cache population
+
+Using remote dev is not subject the Cloudflare API rate limit of 1,200 requests per 5 minutes that caused failures for large applications with thousands of prerendered pages.

packages/cloudflare/src/cli/commands/deploy.ts

@@ -30,19 +30,24 @@ export async function deployCommand(args: WithWranglerArgs<{ cacheChunkSize?: nu
 
 	const envVars = await getEnvFromPlatformProxy(config, buildOpts);
 
-	await populateCache(
-		buildOpts,
-		config,
-		wranglerConfig,
-		{
-			target: "remote",
-			environment: args.env,
-			wranglerConfigPath: args.wranglerConfigPath,
-			cacheChunkSize: args.cacheChunkSize,
-			shouldUsePreviewId: false,
-		},
-		envVars
-	);
+	try {
+		await populateCache(
+			buildOpts,
+			config,
+			wranglerConfig,
+			{
+				target: "remote",
+				environment: args.env,
+				wranglerConfigPath: args.wranglerConfigPath,
+				cacheChunkSize: args.cacheChunkSize,
+				shouldUsePreviewId: false,
+			},
+			envVars
+		);
+	} catch (error) {
+		logger.error(error instanceof Error ? error.message : String(error));
+		process.exit(1);
+	}
 
 	const deploymentMapping = await getDeploymentMapping(buildOpts, config, envVars);
 

packages/cloudflare/src/cli/commands/populate-cache.spec.ts

@@ -2,10 +2,15 @@ import { mkdirSync, writeFileSync } from "node:fs";
 import path from "node:path";
 
 import type { BuildOptions } from "@opennextjs/aws/build/helper.js";
+import { OpenNextConfig } from "@opennextjs/aws/types/open-next.js";
 import mockFs from "mock-fs";
 import { afterAll, afterEach, beforeAll, describe, expect, test, vi } from "vitest";
+import type { Unstable_Config as WranglerConfig } from "wrangler";
+import { unstable_startWorker } from "wrangler";
 
-import { getCacheAssets, populateCache } from "./populate-cache.js";
+import { ensureR2Bucket } from "../utils/ensure-r2-bucket.js";
+import { getCacheAssets, populateCache, PopulateCacheOptions } from "./populate-cache.js";
+import { WorkerEnvVar } from "./utils/helpers.js";
 
 describe("getCacheAssets", () => {
 	beforeAll(() => {
@@ -78,7 +83,37 @@ vi.mock("./utils/helpers.js", () => ({
 	quoteShellMeta: vi.fn((s) => s),
 }));
 
+vi.mock("../utils/ensure-r2-bucket.js");
+vi.mock("wrangler");
+
 describe("populateCache", () => {
+	// @ts-expect-error - Partial mock of OpenNextConfig for testing
+	const buildOptions: BuildOptions = {
+		appPath: "/test/app",
+		outputDir: "/test/output",
+	};
+	const config: OpenNextConfig = {
+		default: {
+			override: {
+				// @ts-expect-error - Use R2 incremental cache
+				incrementalCache: "cf-r2-incremental-cache",
+			},
+		},
+	};
+	// @ts-expect-error - Partial mock of WranglerConfig for testing
+	const wranglerConfig: WranglerConfig = {
+		r2_buckets: [
+			{
+				binding: "NEXT_INC_CACHE_R2_BUCKET",
+				bucket_name: "test-bucket",
+				preview_bucket_name: "preview-bucket",
+				jurisdiction: "eu",
+			},
+		],
+	};
+	// @ts-expect-error - Use partial WorkerEnvVar for testing
+	const envVars: WorkerEnvVar = {};
+
 	const setupMockFileSystem = () => {
 		mockFs({
 			"/test/output": {
@@ -95,85 +130,147 @@ describe("populateCache", () => {
 		});
 	};
 
-	describe.each([{ target: "local" as const }, { target: "remote" as const }])(
-		"R2 incremental cache",
-		({ target }) => {
-			afterEach(() => {
-				mockFs.restore();
-			});
+	describe("R2 incremental cache", () => {
+		afterEach(() => {
+			vi.resetAllMocks();
+			vi.useRealTimers();
+			mockFs.restore();
+		});
 
-			test(target, async () => {
-				const { runWrangler } = await import("./utils/run-wrangler.js");
+		test.each<PopulateCacheOptions>([
+			{ target: "local", shouldUsePreviewId: false },
+			{ target: "remote", shouldUsePreviewId: false },
+			{ target: "remote", shouldUsePreviewId: true },
+		])(
+			`$target (shouldUsePreviewId: $shouldUsePreviewId) - starts worker and sends individual cache entries with the cache key header`,
+			async (populateCacheOptions) => {
+				const bucketName =
+					populateCacheOptions.target === "remote" && populateCacheOptions.shouldUsePreviewId
+						? "preview-bucket"
+						: "test-bucket";
+				const mockWorkerDispose = vi.fn();
 
 				setupMockFileSystem();
-				vi.mocked(runWrangler).mockClear();
-
-				await populateCache(
-					{
-						outputDir: "/test/output",
-					} as BuildOptions,
-					{
-						default: {
-							override: {
-								incrementalCache: "cf-r2-incremental-cache",
-							},
-						},
-					} as any, // eslint-disable-line @typescript-eslint/no-explicit-any
-					{
-						r2_buckets: [
-							{
-								binding: "NEXT_INC_CACHE_R2_BUCKET",
-								bucket_name: "test-bucket",
-							},
-						],
-					} as any, // eslint-disable-line @typescript-eslint/no-explicit-any
-					{ target, shouldUsePreviewId: false },
-					{} as any // eslint-disable-line @typescript-eslint/no-explicit-any
-				);
+				vi.useFakeTimers();
+				// @ts-expect-error - Mock unstable_startWorker to return a mock worker instance
+				vi.mocked(unstable_startWorker).mockResolvedValueOnce({
+					ready: Promise.resolve(),
+					url: Promise.resolve(new URL("http://localhost:12345")),
+					dispose: mockWorkerDispose,
+				});
+				vi.mocked(ensureR2Bucket).mockResolvedValueOnce({ success: true, bucketName });
 
-				expect(runWrangler).toHaveBeenCalledWith(
-					expect.anything(),
-					expect.arrayContaining(["r2 bulk put", "test-bucket"]),
-					expect.objectContaining({ target })
+				// Mock fetch to return a successful response for each individual entry.
+				const fetchMock = vi.spyOn(global, "fetch").mockResolvedValue(
+					new Response(JSON.stringify({ success: true }), {
+						status: 200,
+						headers: { "Content-Type": "application/json" },
+					})
 				);
-			});
 
-			test(`${target} using jurisdiction`, async () => {
-				const { runWrangler } = await import("./utils/run-wrangler.js");
+				await populateCache(buildOptions, config, wranglerConfig, populateCacheOptions, envVars);
 
-				setupMockFileSystem();
-				vi.mocked(runWrangler).mockClear();
-
-				await populateCache(
-					{
-						outputDir: "/test/output",
-					} as BuildOptions,
-					{
-						default: {
-							override: {
-								incrementalCache: "cf-r2-incremental-cache",
-							},
-						},
-					} as any, // eslint-disable-line @typescript-eslint/no-explicit-any
-					{
-						r2_buckets: [
-							{
-								binding: "NEXT_INC_CACHE_R2_BUCKET",
-								bucket_name: "test-bucket",
+				expect(unstable_startWorker).toHaveBeenCalledWith(
+					expect.objectContaining({
+						bindings: expect.objectContaining({
+							R2: expect.objectContaining({
+								type: "r2_bucket",
+								bucket_name: bucketName,
 								jurisdiction: "eu",
-							},
-						],
-					} as any, // eslint-disable-line @typescript-eslint/no-explicit-any
-					{ target, shouldUsePreviewId: false },
-					{} as any // eslint-disable-line @typescript-eslint/no-explicit-any
+							}),
+						}),
+						dev: expect.objectContaining({
+							remote: populateCacheOptions.target === "remote",
+						}),
+					})
 				);
 
-				expect(runWrangler).toHaveBeenCalledWith(
-					expect.anything(),
-					expect.arrayContaining(["r2 bulk put", "test-bucket", "--jurisdiction eu"]),
-					expect.objectContaining({ target })
-				);
+				if (populateCacheOptions.target === "remote") {
+					expect(ensureR2Bucket).toHaveBeenCalledWith("/test/app", bucketName, "eu");
+				} else {
+					expect(ensureR2Bucket).not.toHaveBeenCalled();
+				}
+
+				expect(fetchMock).toBeCalled();
+
+				for (const [input, init] of fetchMock.mock.calls) {
+					expect(input).toBe("http://localhost:12345/populate");
+					expect(init?.method).toBe("POST");
+					expect(init?.headers).toEqual({
+						"x-opennext-cache-key": expect.any(String),
+					});
+					expect(init?.body).toBeInstanceOf(Buffer);
+				}
+
+				// Verify worker was disposed after sending entries.
+				expect(mockWorkerDispose).toHaveBeenCalled();
+			}
+		);
+
+		test("remote - exits when bucket provisioning fails", async () => {
+			setupMockFileSystem();
+			vi.mocked(ensureR2Bucket).mockResolvedValueOnce({
+				success: false,
+				error: "wrangler login failed",
 			});
-		}
-	);
+
+			const result = populateCache(
+				buildOptions,
+				config,
+				wranglerConfig,
+				{ target: "remote", shouldUsePreviewId: false },
+				envVars
+			);
+
+			await expect(result).rejects.toThrow(
+				'Failed to provision remote R2 bucket "test-bucket" for binding "NEXT_INC_CACHE_R2_BUCKET": wrangler login failed'
+			);
+
+			expect(unstable_startWorker).not.toHaveBeenCalled();
+		});
+
+		test("remote - retries timed out requests to the R2 worker", async () => {
+			setupMockFileSystem();
+			vi.useFakeTimers();
+
+			const mockWorkerDispose = vi.fn();
+			// @ts-expect-error - Mock unstable_startWorker to return a mock worker instance
+			vi.mocked(unstable_startWorker).mockResolvedValueOnce({
+				ready: Promise.resolve(),
+				url: Promise.resolve(new URL("http://localhost:12345")),
+				dispose: mockWorkerDispose,
+			});
+			vi.mocked(ensureR2Bucket).mockResolvedValueOnce({
+				success: true,
+				bucketName: "test-bucket",
+			});
+
+			const timeoutError = new Error("Timed out waiting for worker response");
+			timeoutError.name = "TimeoutError";
+
+			const fetchMock = vi
+				.spyOn(global, "fetch")
+				.mockRejectedValueOnce(timeoutError)
+				.mockResolvedValueOnce(
+					new Response(JSON.stringify({ success: true }), {
+						status: 200,
+						headers: { "Content-Type": "application/json" },
+					})
+				);
+
+			const result = populateCache(
+				buildOptions,
+				config,
+				wranglerConfig,
+				{ target: "remote", shouldUsePreviewId: false },
+				envVars
+			);
+
+			await vi.advanceTimersByTimeAsync(250);
+			await result;
+
+			expect(fetchMock).toHaveBeenCalledTimes(2);
+			expect(mockWorkerDispose).toHaveBeenCalled();
+		});
+	});
 });