feat: add opt-in rclone cache population

.changeset/recovered-rclone-batch-upload.md

@@ -0,0 +1,39 @@
+---
+"@opennextjs/cloudflare": minor
+---
+
+feature: add opt-in batch upload via `rclone` for fast R2 cache population.
+
+**Key Changes:**
+
+1. **Optional `rclone` Upload**: Install the optional `rclone.js` peer dependency and pass `--rclone` to opt in to `rclone` based batch uploads.
+
+   - `R2_ACCESS_KEY_ID`
+   - `R2_SECRET_ACCESS_KEY`
+   - `CF_ACCOUNT_ID`
+
+2. **Explicit Opt-in**: The existing worker-based population path remains the default. `rclone` is only loaded when `--rclone` is used for a remote cache.
+
+3. **Clear Errors**: The CLI reports missing credentials or a missing `rclone.js` installation when the option is used.
+
+**Usage:**
+
+Install `rclone.js`, then add the secrets in a `.env`/`.dev.vars` file in your project root:
+
+```bash
+pnpm install rclone.js
+pnpm approve-builds # select rclone.js
+pnpm rebuild rclone.js
+R2_ACCESS_KEY_ID=your_key
+R2_SECRET_ACCESS_KEY=your_secret
+CF_ACCOUNT_ID=your_account
+
+opennextjs-cloudflare deploy --rclone
+```
+
+You can also set the environment variables for CI builds.
+
+**Notes:**
+
+- You can follow documentation <https://developers.cloudflare.com/r2/api/tokens/> for creating API tokens with appropriate permissions for R2 access.
+- `rclone` may not be supported on all platforms.

packages/cloudflare/package.json

@@ -70,6 +70,7 @@
 		"@types/mock-fs": "catalog:",
 		"@types/node": "catalog:",
 		"@types/picomatch": "^4.0.0",
+		"@types/rclone.js": "^0.6.1",
 		"@types/yargs": "catalog:",
 		"diff": "^8.0.2",
 		"esbuild": "catalog:",
@@ -81,13 +82,20 @@
 		"mock-fs": "catalog:",
 		"next": "catalog:",
 		"picomatch": "^4.0.2",
+		"rclone.js": "^0.6.6",
 		"rimraf": "catalog:",
 		"typescript": "catalog:",
 		"typescript-eslint": "catalog:",
 		"vitest": "catalog:"
 	},
 	"peerDependencies": {
 		"next": ">=15.5.18 <16 || >=16.2.6",
+		"rclone.js": "^0.6.6",
 		"wrangler": "catalog:"
+	},
+	"peerDependenciesMeta": {
+		"rclone.js": {
+			"optional": true
+		}
 	}
 }

packages/cloudflare/src/api/cloudflare-context.ts

@@ -83,8 +83,13 @@ declare global {
 		CF_PREVIEW_DOMAIN?: string;
 		// Should have the `Workers Scripts:Read` permission
 		CF_WORKERS_SCRIPTS_API_TOKEN?: string;
-		// Cloudflare account id - needed for skew protection
+
+		// Cloudflare account id - needed for skew protection and R2 batch population
 		CF_ACCOUNT_ID?: string;
+
+		// R2 S3 API credentials used by `rclone`
+		R2_ACCESS_KEY_ID?: string;
+		R2_SECRET_ACCESS_KEY?: string;
 	}
 }
 

packages/cloudflare/src/cli/commands/deploy.ts

@@ -20,7 +20,9 @@ import {
  *
  * @param args
  */
-export async function deployCommand(args: WithWranglerArgs<{ cacheChunkSize?: number }>): Promise<void> {
+export async function deployCommand(
+	args: WithWranglerArgs<{ cacheChunkSize?: number; rclone: boolean }>
+): Promise<void> {
 	printHeaders("deploy");
 
 	const { config } = await retrieveCompiledConfig();
@@ -45,6 +47,7 @@ export async function deployCommand(args: WithWranglerArgs<{ cacheChunkSize?: nu
 			environment: args.env,
 			wranglerConfigPath: args.wranglerConfigPath,
 			cacheChunkSize: args.cacheChunkSize,
+			useRclone: args.rclone,
 			shouldUsePreviewId: false,
 		},
 		envVars

packages/cloudflare/src/cli/commands/populate-cache.spec.ts

@@ -1,8 +1,10 @@
+import { EventEmitter } from "node:events";
 import { mkdirSync, writeFileSync } from "node:fs";
 import path from "node:path";
 
 import type { BuildOptions } from "@opennextjs/aws/build/helper.js";
 import mockFs from "mock-fs";
+import rclone from "rclone.js";
 import { afterAll, afterEach, beforeAll, describe, expect, test, vi } from "vitest";
 import type { Unstable_Config as WranglerConfig } from "wrangler";
 import { unstable_startWorker } from "wrangler";
@@ -12,6 +14,7 @@ import r2IncrementalCache from "../../api/overrides/incremental-cache/r2-increme
 import { ensureR2Bucket } from "../utils/ensure-r2-bucket.js";
 import {
 	getCacheAssets,
+	loadRclone,
 	MAX_REQUEST_RETRIES,
 	MAX_RETRY_DELAY_MS,
 	populateCache,
@@ -104,6 +107,18 @@ vi.mock("./utils/helpers.js", () => ({
 }));
 
 vi.mock("../utils/ensure-r2-bucket.js");
+vi.mock("rclone.js", () => ({
+	default: {
+		copy: vi.fn(() => {
+			const child = new EventEmitter();
+			queueMicrotask(() => child.emit("close", 0, null));
+			return child;
+		}),
+		promises: {
+			version: vi.fn(async () => ""),
+		},
+	},
+}));
 vi.mock("wrangler");
 
 describe("populateCache", () => {
@@ -224,6 +239,153 @@ describe("populateCache", () => {
 			}
 		);
 
+		test("remote with rclone - uploads staged cache entries without starting a worker", async () => {
+			setupMockFileSystem();
+
+			await populateCache(
+				buildOptions,
+				config,
+				wranglerConfig,
+				{ target: "remote", shouldUsePreviewId: false, useRclone: true, cacheChunkSize: 20 },
+				{
+					R2_ACCESS_KEY_ID: "access-key",
+					R2_SECRET_ACCESS_KEY: "secret-key",
+					CF_ACCOUNT_ID: "account-id",
+				} as WorkerEnvVar
+			);
+
+			expect(rclone.copy).toHaveBeenCalledWith(
+				expect.any(String),
+				"r2:test-bucket",
+				expect.objectContaining({
+					progress: true,
+					transfers: 20,
+					checkers: 10,
+					env: expect.objectContaining({ RCLONE_CONFIG: expect.any(String) }),
+					stdio: "inherit",
+				})
+			);
+			expect(unstable_startWorker).not.toHaveBeenCalled();
+			expect(ensureR2Bucket).not.toHaveBeenCalled();
+		});
+
+		test("remote with rclone - uses the default transfer concurrency", async () => {
+			setupMockFileSystem();
+
+			await populateCache(
+				buildOptions,
+				config,
+				wranglerConfig,
+				{ target: "remote", shouldUsePreviewId: false, useRclone: true },
+				{
+					R2_ACCESS_KEY_ID: "access-key",
+					R2_SECRET_ACCESS_KEY: "secret-key",
+					CF_ACCOUNT_ID: "account-id",
+				} as WorkerEnvVar
+			);
+
+			expect(rclone.copy).toHaveBeenCalledWith(
+				expect.any(String),
+				"r2:test-bucket",
+				expect.objectContaining({ transfers: 25, checkers: 12 })
+			);
+		});
+
+		test("remote with rclone - rejects a failed rclone process", async () => {
+			setupMockFileSystem();
+			vi.mocked(rclone.copy).mockImplementationOnce(() => {
+				const child = new EventEmitter();
+				queueMicrotask(() => child.emit("close", 7, null));
+				return child as ReturnType<typeof rclone.copy>;
+			});
+
+			await expect(
+				populateCache(
+					buildOptions,
+					config,
+					wranglerConfig,
+					{ target: "remote", shouldUsePreviewId: false, useRclone: true },
+					{
+						R2_ACCESS_KEY_ID: "access-key",
+						R2_SECRET_ACCESS_KEY: "secret-key",
+						CF_ACCOUNT_ID: "account-id",
+					} as WorkerEnvVar
+				)
+			).rejects.toThrow("rclone exited with code 7");
+		});
+
+		test("local with rclone - rejects the unsupported option", async () => {
+			setupMockFileSystem();
+
+			await expect(
+				populateCache(
+					buildOptions,
+					config,
+					wranglerConfig,
+					{ target: "local", shouldUsePreviewId: false, useRclone: true },
+					envVars
+				)
+			).rejects.toThrow("The `--rclone` option can only be used when populating a remote R2 cache.");
+		});
+
+		test("remote with rclone - rejects missing R2 credentials", async () => {
+			setupMockFileSystem();
+
+			await expect(
+				populateCache(
+					buildOptions,
+					config,
+					wranglerConfig,
+					{ target: "remote", shouldUsePreviewId: false, useRclone: true },
+					envVars
+				)
+			).rejects.toThrow(
+				"R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY, and CF_ACCOUNT_ID must be provided to use `rclone`"
+			);
+		});
+
+		test("remote with rclone - rejects cache prefixes that escape the staging directory", async () => {
+			setupMockFileSystem();
+
+			await expect(
+				populateCache(
+					buildOptions,
+					config,
+					wranglerConfig,
+					{ target: "remote", shouldUsePreviewId: false, useRclone: true },
+					{
+						R2_ACCESS_KEY_ID: "access-key",
+						R2_SECRET_ACCESS_KEY: "secret-key",
+						CF_ACCOUNT_ID: "account-id",
+						NEXT_INC_CACHE_R2_PREFIX: "../escaped",
+					} as WorkerEnvVar
+				)
+			).rejects.toThrow("Cannot stage R2 cache key outside the temporary directory");
+		});
+
+		test("remote with rclone - explains how to install the rclone executable with pnpm", async () => {
+			setupMockFileSystem();
+			vi.mocked(rclone.promises.version).mockRejectedValueOnce(
+				Object.assign(new Error("spawn rclone ENOENT"), { code: "ENOENT" })
+			);
+
+			await expect(
+				populateCache(
+					buildOptions,
+					config,
+					wranglerConfig,
+					{ target: "remote", shouldUsePreviewId: false, useRclone: true },
+					{
+						R2_ACCESS_KEY_ID: "access-key",
+						R2_SECRET_ACCESS_KEY: "secret-key",
+						CF_ACCOUNT_ID: "account-id",
+					} as WorkerEnvVar
+				)
+			).rejects.toThrow(
+				"pnpm users must allow its install script with `pnpm approve-builds`, select `rclone.js`, then run `pnpm rebuild rclone.js`"
+			);
+		});
+
 		test("remote - exits when bucket provisioning fails", async () => {
 			setupMockFileSystem();
 			vi.mocked(ensureR2Bucket).mockResolvedValueOnce({
@@ -465,3 +627,19 @@ describe("populateCache", () => {
 		});
 	});
 });
+
+describe("loadRclone", () => {
+	test("reports how to install the optional peer dependency", async () => {
+		const missingModuleError = Object.assign(new Error("Cannot find package 'rclone.js'"), {
+			code: "ERR_MODULE_NOT_FOUND",
+		});
+
+		await expect(
+			loadRclone(async () => {
+				throw missingModuleError;
+			})
+		).rejects.toThrow(
+			"The `--rclone` option requires the optional `rclone.js` peer dependency. Install it in your project before using this option."
+		);
+	});
+});