Add an rclone based option to populate the R2 cache

.changeset/recovered-rclone-batch-upload.md

@@ -0,0 +1,48 @@
+---
+"@opennextjs/cloudflare": minor
+---
+
+feature: optional batch upload for faster R2 cache population
+
+This update recovers optional batch upload support for R2 cache population, significantly improving upload performance for large caches when enabled via .env or environment variables.
+
+**Key Changes:**
+
+1. **Optional Batch Upload**: Configure R2 credentials via .env or environment variables to enable faster batch uploads:
+
+   - `R2_ACCESS_KEY_ID`
+   - `R2_SECRET_ACCESS_KEY`
+   - `CF_ACCOUNT_ID`
+
+2. **Automatic Detection**: When credentials are detected, batch upload is automatically used for better performance
+
+3. **Smart Fallback**: If credentials are not configured, the CLI falls back to standard Wrangler uploads with a helpful message about enabling batch upload for better performance
+
+**All deployment commands support batch upload:**
+
+- `populateCache` - Explicit cache population
+- `deploy` - Deploy with cache population
+- `upload` - Upload version with cache population
+- `preview` - Preview with cache population
+
+**Performance Benefits (when batch upload is enabled):**
+
+- Parallel transfer capabilities (32 concurrent transfers)
+- Significantly faster for large caches
+- Reduced API calls to Cloudflare
+
+**Usage:**
+
+Add the credentials in a `.env`/`.dev.vars` file in your project root:
+
+```bash
+R2_ACCESS_KEY_ID=your_key
+R2_SECRET_ACCESS_KEY=your_secret
+CF_ACCOUNT_ID=your_account
+```
+
+You can also set the environment variables for CI builds.
+
+**Note:**
+
+You can follow documentation https://developers.cloudflare.com/r2/api/tokens/ for creating API tokens with appropriate permissions for R2 access.

packages/cloudflare/package.json

@@ -58,6 +58,7 @@
 		"cloudflare": "^4.4.1",
 		"enquirer": "^2.4.1",
 		"glob": "catalog:",
+		"rclone.js": "^0.6.6",
 		"ts-tqdm": "^0.8.6",
 		"yargs": "catalog:"
 	},
@@ -68,6 +69,7 @@
 		"@types/mock-fs": "catalog:",
 		"@types/node": "catalog:",
 		"@types/picomatch": "^4.0.0",
+		"@types/rclone.js": "^0.6.1",
 		"@types/yargs": "catalog:",
 		"diff": "^8.0.2",
 		"esbuild": "catalog:",
@@ -88,4 +90,4 @@
 		"wrangler": "catalog:",
 		"next": "~15.0.8 || ~15.1.12 || ~15.2.9 || ~15.3.9 || ~15.4.11 || ~15.5.10 || ~16.0.11 || ^16.1.5"
 	}
-}
+}

packages/cloudflare/src/api/cloudflare-context.ts

@@ -83,8 +83,13 @@ declare global {
 		CF_PREVIEW_DOMAIN?: string;
 		// Should have the `Workers Scripts:Read` permission
 		CF_WORKERS_SCRIPTS_API_TOKEN?: string;
-		// Cloudflare account id - needed for skew protection
+
+		// Cloudflare account id - needed for skew protection and R2 batch population
 		CF_ACCOUNT_ID?: string;
+
+		// R2 API credentials for batch cache population (optional, enables faster uploads)
+		R2_ACCESS_KEY_ID?: string;
+		R2_SECRET_ACCESS_KEY?: string;
 	}
 }
 
@@ -200,10 +205,10 @@ function getCloudflareContextSync<
 	if (inSSG()) {
 		throw new Error(
 			`\n\nERROR: \`getCloudflareContext\` has been called in sync mode in either a static route or at the top level of a non-static one,` +
-				` both cases are not allowed but can be solved by either:\n` +
-				`  - make sure that the call is not at the top level and that the route is not static\n` +
-				`  - call \`getCloudflareContext({async: true})\` to use the \`async\` mode\n` +
-				`  - avoid calling \`getCloudflareContext\` in the route\n`
+			` both cases are not allowed but can be solved by either:\n` +
+			`  - make sure that the call is not at the top level and that the route is not static\n` +
+			`  - call \`getCloudflareContext({async: true})\` to use the \`async\` mode\n` +
+			`  - avoid calling \`getCloudflareContext\` in the route\n`
 		);
 	}
 
@@ -252,7 +257,7 @@ export async function initOpenNextCloudflareForDev(options?: GetPlatformProxyOpt
 	if (options?.environment && process.env.NEXT_DEV_WRANGLER_ENV) {
 		console.warn(
 			`'initOpenNextCloudflareForDev' has been called with an environment option while NEXT_DEV_WRANGLER_ENV is set.` +
-				` NEXT_DEV_WRANGLER_ENV will be ignored and the environment will be set to: '${options.environment}'`
+			` NEXT_DEV_WRANGLER_ENV will be ignored and the environment will be set to: '${options.environment}'`
 		);
 	}
 

packages/cloudflare/src/cli/commands/populate-cache.spec.ts

@@ -78,7 +78,44 @@ vi.mock("./helpers.js", () => ({
 	quoteShellMeta: vi.fn((s) => s),
 }));
 
+// Mock rclone.js promises API to simulate successful copy operations by default
+vi.mock("rclone.js", () => ({
+	default: {
+		promises: {
+			copy: vi.fn(() => Promise.resolve("")),
+		},
+	},
+}));
+
 describe("populateCache", () => {
+	// Test fixtures
+	const createTestBuildOptions = (): BuildOptions =>
+		({
+			outputDir: "/test/output",
+		}) as BuildOptions;
+
+	const createTestOpenNextConfig = () => ({
+		default: {
+			override: {
+				incrementalCache: "cf-r2-incremental-cache",
+			},
+		},
+	});
+
+	const createTestWranglerConfig = () => ({
+		r2_buckets: [
+			{
+				binding: "NEXT_INC_CACHE_R2_BUCKET",
+				bucket_name: "test-bucket",
+			},
+		],
+	});
+
+	const createTestPopulateCacheOptions = () => ({
+		target: "local" as const,
+		shouldUsePreviewId: false,
+	});
+
 	const setupMockFileSystem = () => {
 		mockFs({
 			"/test/output": {
@@ -95,85 +132,153 @@ describe("populateCache", () => {
 		});
 	};
 
-	describe.each([{ target: "local" as const }, { target: "remote" as const }])(
-		"R2 incremental cache",
-		({ target }) => {
-			afterEach(() => {
-				mockFs.restore();
-			});
-
-			test(target, async () => {
-				const { runWrangler } = await import("../utils/run-wrangler.js");
-
-				setupMockFileSystem();
-				vi.mocked(runWrangler).mockClear();
-
-				await populateCache(
-					{
-						outputDir: "/test/output",
-					} as BuildOptions,
-					{
-						default: {
-							override: {
-								incrementalCache: "cf-r2-incremental-cache",
-							},
-						},
-					} as any, // eslint-disable-line @typescript-eslint/no-explicit-any
-					{
-						r2_buckets: [
-							{
-								binding: "NEXT_INC_CACHE_R2_BUCKET",
-								bucket_name: "test-bucket",
-							},
-						],
-					} as any, // eslint-disable-line @typescript-eslint/no-explicit-any
-					{ target, shouldUsePreviewId: false },
-					{} as any // eslint-disable-line @typescript-eslint/no-explicit-any
-				);
-
-				expect(runWrangler).toHaveBeenCalledWith(
-					expect.anything(),
-					expect.arrayContaining(["r2 bulk put", "test-bucket"]),
-					expect.objectContaining({ target })
-				);
-			});
-
-			test(`${target} using jurisdiction`, async () => {
-				const { runWrangler } = await import("../utils/run-wrangler.js");
-
-				setupMockFileSystem();
-				vi.mocked(runWrangler).mockClear();
-
-				await populateCache(
-					{
-						outputDir: "/test/output",
-					} as BuildOptions,
-					{
-						default: {
-							override: {
-								incrementalCache: "cf-r2-incremental-cache",
-							},
-						},
-					} as any, // eslint-disable-line @typescript-eslint/no-explicit-any
-					{
-						r2_buckets: [
-							{
-								binding: "NEXT_INC_CACHE_R2_BUCKET",
-								bucket_name: "test-bucket",
-								jurisdiction: "eu",
-							},
-						],
-					} as any, // eslint-disable-line @typescript-eslint/no-explicit-any
-					{ target, shouldUsePreviewId: false },
-					{} as any // eslint-disable-line @typescript-eslint/no-explicit-any
-				);
-
-				expect(runWrangler).toHaveBeenCalledWith(
-					expect.anything(),
-					expect.arrayContaining(["r2 bulk put", "test-bucket", "--jurisdiction eu"]),
-					expect.objectContaining({ target })
-				);
-			});
-		}
-	);
+	describe("R2 incremental cache", () => {
+		afterEach(() => {
+			mockFs.restore();
+			vi.unstubAllEnvs();
+		});
+
+		test("uses sequential upload for local target (skips batch upload)", async () => {
+			const { runWrangler } = await import("../utils/run-wrangler.js");
+			const rcloneModule = (await import("rclone.js")).default;
+
+			setupMockFileSystem();
+			vi.mocked(runWrangler).mockClear();
+			vi.mocked(rcloneModule.promises.copy).mockClear();
+
+			// Test with local target - should skip batch upload even with credentials
+			await populateCache(
+				createTestBuildOptions(),
+				createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+				createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+				{ target: "local" as const, shouldUsePreviewId: false },
+				{
+					R2_ACCESS_KEY_ID: "test_access_key",
+					R2_SECRET_ACCESS_KEY: "test_secret_key",
+					CF_ACCOUNT_ID: "test_account_id",
+				} as any // eslint-disable-line @typescript-eslint/no-explicit-any
+			);
+
+			// Should use sequential upload (runWrangler), not batch upload (rclone.js)
+			expect(runWrangler).toHaveBeenCalled();
+			expect(rcloneModule.promises.copy).not.toHaveBeenCalled();
+		});
+
+		test("uses sequential upload when R2 credentials are not provided", async () => {
+			const { runWrangler } = await import("../utils/run-wrangler.js");
+			const rcloneModule = (await import("rclone.js")).default;
+
+			setupMockFileSystem();
+			vi.mocked(runWrangler).mockClear();
+			vi.mocked(rcloneModule.promises.copy).mockClear();
+
+			// Test uses partial types for simplicity - full config not needed
+			// Pass empty envVars to simulate no R2 credentials
+			await populateCache(
+				createTestBuildOptions(),
+				createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+				createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+				createTestPopulateCacheOptions(),
+				{} as any // eslint-disable-line @typescript-eslint/no-explicit-any
+			);
+
+			expect(runWrangler).toHaveBeenCalled();
+			expect(rcloneModule.promises.copy).not.toHaveBeenCalled();
+		});
+
+		test("uses batch upload with temporary config for remote target when R2 credentials are provided", async () => {
+			const rcloneModule = (await import("rclone.js")).default;
+
+			setupMockFileSystem();
+			vi.mocked(rcloneModule.promises.copy).mockClear();
+
+			// Test uses partial types for simplicity - full config not needed
+			// Pass envVars with R2 credentials and remote target to enable batch upload
+			await populateCache(
+				createTestBuildOptions(),
+				createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+				createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+				{ target: "remote" as const, shouldUsePreviewId: false },
+				{
+					R2_ACCESS_KEY_ID: "test_access_key",
+					R2_SECRET_ACCESS_KEY: "test_secret_key",
+					CF_ACCOUNT_ID: "test_account_id",
+				} as any // eslint-disable-line @typescript-eslint/no-explicit-any
+			);
+
+			// Verify batch upload was used with correct parameters and temporary config
+			expect(rcloneModule.promises.copy).toHaveBeenCalledWith(
+				expect.any(String), // staging directory
+				"r2:test-bucket",
+				expect.objectContaining({
+					progress: true,
+					transfers: 16,
+					checkers: 8,
+					env: expect.objectContaining({
+						RCLONE_CONFIG: expect.stringMatching(/rclone-config-\d+\.conf$/),
+					}),
+				})
+			);
+		});
+
+		test("handles rclone errors with status > 0 for remote target", async () => {
+			const { runWrangler } = await import("../utils/run-wrangler.js");
+			const rcloneModule = (await import("rclone.js")).default;
+
+			setupMockFileSystem();
+
+			// Mock rclone failure - Promise rejection
+			vi.mocked(rcloneModule.promises.copy).mockRejectedValueOnce(
+				new Error("rclone copy failed with exit code 7")
+			);
+
+			vi.mocked(runWrangler).mockClear();
+
+			// Pass envVars with R2 credentials and remote target to enable batch upload (which will fail)
+			await populateCache(
+				createTestBuildOptions(),
+				createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+				createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+				{ target: "remote" as const, shouldUsePreviewId: false },
+				{
+					R2_ACCESS_KEY_ID: "test_access_key",
+					R2_SECRET_ACCESS_KEY: "test_secret_key",
+					CF_ACCOUNT_ID: "test_account_id",
+				} as any // eslint-disable-line @typescript-eslint/no-explicit-any
+			);
+
+			// Should fall back to sequential upload when batch upload fails
+			expect(runWrangler).toHaveBeenCalled();
+		});
+
+		test("handles rclone errors with stderr output for remote target", async () => {
+			const { runWrangler } = await import("../utils/run-wrangler.js");
+			const rcloneModule = (await import("rclone.js")).default;
+
+			setupMockFileSystem();
+
+			// Mock rclone error - Promise rejection with stderr message
+			vi.mocked(rcloneModule.promises.copy).mockRejectedValueOnce(
+				new Error("ERROR : Failed to copy: AccessDenied: Access Denied (403)")
+			);
+
+			vi.mocked(runWrangler).mockClear();
+
+			// Pass envVars with R2 credentials and remote target to enable batch upload (which will fail)
+			await populateCache(
+				createTestBuildOptions(),
+				createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+				createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any
+				{ target: "remote" as const, shouldUsePreviewId: false },
+				{
+					R2_ACCESS_KEY_ID: "test_access_key",
+					R2_SECRET_ACCESS_KEY: "test_secret_key",
+					CF_ACCOUNT_ID: "test_account_id",
+				} as any // eslint-disable-line @typescript-eslint/no-explicit-any
+			);
+
+			// Should fall back to standard upload when batch upload fails
+			expect(runWrangler).toHaveBeenCalled();
+		});
+	});
 });