From f5eee95b5264cfd7006321f318c76b89aac68eb5 Mon Sep 17 00:00:00 2001 From: Mateusz Grab Date: Thu, 5 Feb 2026 19:32:53 +0100 Subject: [PATCH 01/10] Recover #925 PR concept for projects with huge R2 incremental cache --- .changeset/recovered-rclone-batch-upload.md | 48 ++++ packages/cloudflare/package.json | 4 +- .../cloudflare/src/api/cloudflare-context.ts | 17 +- .../src/cli/commands/populate-cache.spec.ts | 267 ++++++++++++------ .../src/cli/commands/populate-cache.ts | 226 +++++++++++++-- pnpm-lock.yaml | 176 +++++------- 6 files changed, 516 insertions(+), 222 deletions(-) create mode 100644 .changeset/recovered-rclone-batch-upload.md diff --git a/.changeset/recovered-rclone-batch-upload.md b/.changeset/recovered-rclone-batch-upload.md new file mode 100644 index 000000000..d15e9d315 --- /dev/null +++ b/.changeset/recovered-rclone-batch-upload.md @@ -0,0 +1,48 @@ +--- +"@opennextjs/cloudflare": minor +--- + +feature: optional batch upload for faster R2 cache population + +This update recovers optional batch upload support for R2 cache population, significantly improving upload performance for large caches when enabled via .env or environment variables. + +**Key Changes:** + +1. **Optional Batch Upload**: Configure R2 credentials via .env or environment variables to enable faster batch uploads: + + - `R2_ACCESS_KEY_ID` + - `R2_SECRET_ACCESS_KEY` + - `CF_ACCOUNT_ID` + +2. **Automatic Detection**: When credentials are detected, batch upload is automatically used for better performance + +3. **Smart Fallback**: If credentials are not configured, the CLI falls back to standard Wrangler uploads with a helpful message about enabling batch upload for better performance + +**All deployment commands support batch upload:** + +- `populateCache` - Explicit cache population +- `deploy` - Deploy with cache population +- `upload` - Upload version with cache population +- `preview` - Preview with cache population + +**Performance Benefits (when batch upload is enabled):** + +- Parallel transfer capabilities (32 concurrent transfers) +- Significantly faster for large caches +- Reduced API calls to Cloudflare + +**Usage:** + +Add the credentials in a `.env`/`.dev.vars` file in your project root: + +```bash +R2_ACCESS_KEY_ID=your_key +R2_SECRET_ACCESS_KEY=your_secret +CF_ACCOUNT_ID=your_account +``` + +You can also set the environment variables for CI builds. + +**Note:** + +You can follow documentation https://developers.cloudflare.com/r2/api/tokens/ for creating API tokens with appropriate permissions for R2 access. diff --git a/packages/cloudflare/package.json b/packages/cloudflare/package.json index c7830b029..271dfa9d0 100644 --- a/packages/cloudflare/package.json +++ b/packages/cloudflare/package.json @@ -58,6 +58,7 @@ "cloudflare": "^4.4.1", "enquirer": "^2.4.1", "glob": "catalog:", + "rclone.js": "^0.6.6", "ts-tqdm": "^0.8.6", "yargs": "catalog:" }, @@ -68,6 +69,7 @@ "@types/mock-fs": "catalog:", "@types/node": "catalog:", "@types/picomatch": "^4.0.0", + "@types/rclone.js": "^0.6.1", "@types/yargs": "catalog:", "diff": "^8.0.2", "esbuild": "catalog:", @@ -88,4 +90,4 @@ "wrangler": "catalog:", "next": "~15.0.8 || ~15.1.12 || ~15.2.9 || ~15.3.9 || ~15.4.11 || ~15.5.10 || ~16.0.11 || ^16.1.5" } -} +} \ No newline at end of file diff --git a/packages/cloudflare/src/api/cloudflare-context.ts b/packages/cloudflare/src/api/cloudflare-context.ts index cfde3167b..2975b3a33 100644 --- a/packages/cloudflare/src/api/cloudflare-context.ts +++ b/packages/cloudflare/src/api/cloudflare-context.ts @@ -83,8 +83,13 @@ declare global { CF_PREVIEW_DOMAIN?: string; // Should have the `Workers Scripts:Read` permission CF_WORKERS_SCRIPTS_API_TOKEN?: string; - // Cloudflare account id - needed for skew protection + + // Cloudflare account id - needed for skew protection and R2 batch population CF_ACCOUNT_ID?: string; + + // R2 API credentials for batch cache population (optional, enables faster uploads) + R2_ACCESS_KEY_ID?: string; + R2_SECRET_ACCESS_KEY?: string; } } @@ -200,10 +205,10 @@ function getCloudflareContextSync< if (inSSG()) { throw new Error( `\n\nERROR: \`getCloudflareContext\` has been called in sync mode in either a static route or at the top level of a non-static one,` + - ` both cases are not allowed but can be solved by either:\n` + - ` - make sure that the call is not at the top level and that the route is not static\n` + - ` - call \`getCloudflareContext({async: true})\` to use the \`async\` mode\n` + - ` - avoid calling \`getCloudflareContext\` in the route\n` + ` both cases are not allowed but can be solved by either:\n` + + ` - make sure that the call is not at the top level and that the route is not static\n` + + ` - call \`getCloudflareContext({async: true})\` to use the \`async\` mode\n` + + ` - avoid calling \`getCloudflareContext\` in the route\n` ); } @@ -252,7 +257,7 @@ export async function initOpenNextCloudflareForDev(options?: GetPlatformProxyOpt if (options?.environment && process.env.NEXT_DEV_WRANGLER_ENV) { console.warn( `'initOpenNextCloudflareForDev' has been called with an environment option while NEXT_DEV_WRANGLER_ENV is set.` + - ` NEXT_DEV_WRANGLER_ENV will be ignored and the environment will be set to: '${options.environment}'` + ` NEXT_DEV_WRANGLER_ENV will be ignored and the environment will be set to: '${options.environment}'` ); } diff --git a/packages/cloudflare/src/cli/commands/populate-cache.spec.ts b/packages/cloudflare/src/cli/commands/populate-cache.spec.ts index 31f54d35a..6743d5bc5 100644 --- a/packages/cloudflare/src/cli/commands/populate-cache.spec.ts +++ b/packages/cloudflare/src/cli/commands/populate-cache.spec.ts @@ -78,7 +78,44 @@ vi.mock("./helpers.js", () => ({ quoteShellMeta: vi.fn((s) => s), })); +// Mock rclone.js promises API to simulate successful copy operations by default +vi.mock("rclone.js", () => ({ + default: { + promises: { + copy: vi.fn(() => Promise.resolve("")), + }, + }, +})); + describe("populateCache", () => { + // Test fixtures + const createTestBuildOptions = (): BuildOptions => + ({ + outputDir: "/test/output", + }) as BuildOptions; + + const createTestOpenNextConfig = () => ({ + default: { + override: { + incrementalCache: "cf-r2-incremental-cache", + }, + }, + }); + + const createTestWranglerConfig = () => ({ + r2_buckets: [ + { + binding: "NEXT_INC_CACHE_R2_BUCKET", + bucket_name: "test-bucket", + }, + ], + }); + + const createTestPopulateCacheOptions = () => ({ + target: "local" as const, + shouldUsePreviewId: false, + }); + const setupMockFileSystem = () => { mockFs({ "/test/output": { @@ -95,85 +132,153 @@ describe("populateCache", () => { }); }; - describe.each([{ target: "local" as const }, { target: "remote" as const }])( - "R2 incremental cache", - ({ target }) => { - afterEach(() => { - mockFs.restore(); - }); - - test(target, async () => { - const { runWrangler } = await import("../utils/run-wrangler.js"); - - setupMockFileSystem(); - vi.mocked(runWrangler).mockClear(); - - await populateCache( - { - outputDir: "/test/output", - } as BuildOptions, - { - default: { - override: { - incrementalCache: "cf-r2-incremental-cache", - }, - }, - } as any, // eslint-disable-line @typescript-eslint/no-explicit-any - { - r2_buckets: [ - { - binding: "NEXT_INC_CACHE_R2_BUCKET", - bucket_name: "test-bucket", - }, - ], - } as any, // eslint-disable-line @typescript-eslint/no-explicit-any - { target, shouldUsePreviewId: false }, - {} as any // eslint-disable-line @typescript-eslint/no-explicit-any - ); - - expect(runWrangler).toHaveBeenCalledWith( - expect.anything(), - expect.arrayContaining(["r2 bulk put", "test-bucket"]), - expect.objectContaining({ target }) - ); - }); - - test(`${target} using jurisdiction`, async () => { - const { runWrangler } = await import("../utils/run-wrangler.js"); - - setupMockFileSystem(); - vi.mocked(runWrangler).mockClear(); - - await populateCache( - { - outputDir: "/test/output", - } as BuildOptions, - { - default: { - override: { - incrementalCache: "cf-r2-incremental-cache", - }, - }, - } as any, // eslint-disable-line @typescript-eslint/no-explicit-any - { - r2_buckets: [ - { - binding: "NEXT_INC_CACHE_R2_BUCKET", - bucket_name: "test-bucket", - jurisdiction: "eu", - }, - ], - } as any, // eslint-disable-line @typescript-eslint/no-explicit-any - { target, shouldUsePreviewId: false }, - {} as any // eslint-disable-line @typescript-eslint/no-explicit-any - ); - - expect(runWrangler).toHaveBeenCalledWith( - expect.anything(), - expect.arrayContaining(["r2 bulk put", "test-bucket", "--jurisdiction eu"]), - expect.objectContaining({ target }) - ); - }); - } - ); + describe("R2 incremental cache", () => { + afterEach(() => { + mockFs.restore(); + vi.unstubAllEnvs(); + }); + + test("uses sequential upload for local target (skips batch upload)", async () => { + const { runWrangler } = await import("../utils/run-wrangler.js"); + const rcloneModule = (await import("rclone.js")).default; + + setupMockFileSystem(); + vi.mocked(runWrangler).mockClear(); + vi.mocked(rcloneModule.promises.copy).mockClear(); + + // Test with local target - should skip batch upload even with credentials + await populateCache( + createTestBuildOptions(), + createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any + createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any + { target: "local" as const, shouldUsePreviewId: false }, + { + R2_ACCESS_KEY_ID: "test_access_key", + R2_SECRET_ACCESS_KEY: "test_secret_key", + CF_ACCOUNT_ID: "test_account_id", + } as any // eslint-disable-line @typescript-eslint/no-explicit-any + ); + + // Should use sequential upload (runWrangler), not batch upload (rclone.js) + expect(runWrangler).toHaveBeenCalled(); + expect(rcloneModule.promises.copy).not.toHaveBeenCalled(); + }); + + test("uses sequential upload when R2 credentials are not provided", async () => { + const { runWrangler } = await import("../utils/run-wrangler.js"); + const rcloneModule = (await import("rclone.js")).default; + + setupMockFileSystem(); + vi.mocked(runWrangler).mockClear(); + vi.mocked(rcloneModule.promises.copy).mockClear(); + + // Test uses partial types for simplicity - full config not needed + // Pass empty envVars to simulate no R2 credentials + await populateCache( + createTestBuildOptions(), + createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any + createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any + createTestPopulateCacheOptions(), + {} as any // eslint-disable-line @typescript-eslint/no-explicit-any + ); + + expect(runWrangler).toHaveBeenCalled(); + expect(rcloneModule.promises.copy).not.toHaveBeenCalled(); + }); + + test("uses batch upload with temporary config for remote target when R2 credentials are provided", async () => { + const rcloneModule = (await import("rclone.js")).default; + + setupMockFileSystem(); + vi.mocked(rcloneModule.promises.copy).mockClear(); + + // Test uses partial types for simplicity - full config not needed + // Pass envVars with R2 credentials and remote target to enable batch upload + await populateCache( + createTestBuildOptions(), + createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any + createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any + { target: "remote" as const, shouldUsePreviewId: false }, + { + R2_ACCESS_KEY_ID: "test_access_key", + R2_SECRET_ACCESS_KEY: "test_secret_key", + CF_ACCOUNT_ID: "test_account_id", + } as any // eslint-disable-line @typescript-eslint/no-explicit-any + ); + + // Verify batch upload was used with correct parameters and temporary config + expect(rcloneModule.promises.copy).toHaveBeenCalledWith( + expect.any(String), // staging directory + "r2:test-bucket", + expect.objectContaining({ + progress: true, + transfers: 16, + checkers: 8, + env: expect.objectContaining({ + RCLONE_CONFIG: expect.stringMatching(/rclone-config-\d+\.conf$/), + }), + }) + ); + }); + + test("handles rclone errors with status > 0 for remote target", async () => { + const { runWrangler } = await import("../utils/run-wrangler.js"); + const rcloneModule = (await import("rclone.js")).default; + + setupMockFileSystem(); + + // Mock rclone failure - Promise rejection + vi.mocked(rcloneModule.promises.copy).mockRejectedValueOnce( + new Error("rclone copy failed with exit code 7") + ); + + vi.mocked(runWrangler).mockClear(); + + // Pass envVars with R2 credentials and remote target to enable batch upload (which will fail) + await populateCache( + createTestBuildOptions(), + createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any + createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any + { target: "remote" as const, shouldUsePreviewId: false }, + { + R2_ACCESS_KEY_ID: "test_access_key", + R2_SECRET_ACCESS_KEY: "test_secret_key", + CF_ACCOUNT_ID: "test_account_id", + } as any // eslint-disable-line @typescript-eslint/no-explicit-any + ); + + // Should fall back to sequential upload when batch upload fails + expect(runWrangler).toHaveBeenCalled(); + }); + + test("handles rclone errors with stderr output for remote target", async () => { + const { runWrangler } = await import("../utils/run-wrangler.js"); + const rcloneModule = (await import("rclone.js")).default; + + setupMockFileSystem(); + + // Mock rclone error - Promise rejection with stderr message + vi.mocked(rcloneModule.promises.copy).mockRejectedValueOnce( + new Error("ERROR : Failed to copy: AccessDenied: Access Denied (403)") + ); + + vi.mocked(runWrangler).mockClear(); + + // Pass envVars with R2 credentials and remote target to enable batch upload (which will fail) + await populateCache( + createTestBuildOptions(), + createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any + createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any + { target: "remote" as const, shouldUsePreviewId: false }, + { + R2_ACCESS_KEY_ID: "test_access_key", + R2_SECRET_ACCESS_KEY: "test_secret_key", + CF_ACCOUNT_ID: "test_account_id", + } as any // eslint-disable-line @typescript-eslint/no-explicit-any + ); + + // Should fall back to standard upload when batch upload fails + expect(runWrangler).toHaveBeenCalled(); + }); + }); }); diff --git a/packages/cloudflare/src/cli/commands/populate-cache.ts b/packages/cloudflare/src/cli/commands/populate-cache.ts index bf4060b00..d9e7dab81 100644 --- a/packages/cloudflare/src/cli/commands/populate-cache.ts +++ b/packages/cloudflare/src/cli/commands/populate-cache.ts @@ -1,6 +1,5 @@ -import fs from "node:fs"; -import fsp from "node:fs/promises"; -import os from "node:os"; +import { copyFileSync, cpSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; import path from "node:path"; import type { BuildOptions } from "@opennextjs/aws/build/helper.js"; @@ -13,6 +12,7 @@ import type { } from "@opennextjs/aws/types/open-next.js"; import type { IncrementalCache, TagCache } from "@opennextjs/aws/types/overrides.js"; import { globSync } from "glob"; +import rclone from "rclone.js"; import { tqdm } from "ts-tqdm"; import type { Unstable_Config as WranglerConfig } from "wrangler"; import type yargs from "yargs"; @@ -91,7 +91,7 @@ export async function populateCache( ) { const { incrementalCache, tagCache } = config.default.override ?? {}; - if (!fs.existsSync(buildOpts.outputDir)) { + if (!existsSync(buildOpts.outputDir)) { logger.error("Unable to populate cache: Open Next build not found"); process.exit(1); } @@ -207,29 +207,141 @@ type PopulateCacheOptions = { shouldUsePreviewId: boolean; }; -async function populateR2IncrementalCache( - buildOpts: BuildOptions, - config: WranglerConfig, - populateCacheOptions: PopulateCacheOptions, +/** + * Create a temporary configuration file for batch upload from environment variables + * @returns Path to the temporary config file or null if env vars not available + */ +function createTempRcloneConfig(accessKey: string, secretKey: string, accountId: string): string | null { + const tempDir = tmpdir(); + const tempConfigPath = path.join(tempDir, `rclone-config-${Date.now()}.conf`); + + const configContent = `[r2] +type = s3 +provider = Cloudflare +access_key_id = ${accessKey} +secret_access_key = ${secretKey} +endpoint = https://${accountId}.r2.cloudflarestorage.com +acl = private +`; + + /** + * 0o600 is an octal number (the 0o prefix indicates octal in JavaScript) + * that represents Unix file permissions: + * + * - 6 (owner): read (4) + write (2) = readable and writable by the file owner + * - 0 (group): no permissions for the group + * - 0 (others): no permissions for anyone else + * + * In symbolic notation, this is: rw------- + */ + writeFileSync(tempConfigPath, configContent, { mode: 0o600 }); + return tempConfigPath; +} + +/** + * Populate R2 incremental cache using batch upload for better performance + * Uses parallel transfers to significantly speed up cache population + */ +async function populateR2IncrementalCacheWithBatchUpload( + bucket: string, + prefix: string | undefined, + assets: CacheAsset[], envVars: WorkerEnvVar ) { - logger.info("\nPopulating R2 incremental cache..."); + const accessKey = envVars.R2_ACCESS_KEY_ID || null; + const secretKey = envVars.R2_SECRET_ACCESS_KEY || null; + const accountId = envVars.CF_ACCOUNT_ID || null; + + // Ensure all required env vars are set correctly + if (!accessKey || !secretKey || !accountId) { + throw new Error( + "Please set R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY, and CF_ACCOUNT_ID environment variables to enable batch upload for remote R2 that bypasses Account Rate Limits." + ); + } - const binding = config.r2_buckets.find( - ({ binding }: { binding: string }) => binding === R2_CACHE_BINDING_NAME - ); - if (!binding) { - throw new Error(`No R2 binding ${JSON.stringify(R2_CACHE_BINDING_NAME)} found!`); + logger.info("\nPopulating remote R2 incremental cache using batch upload..."); + + // Create temporary config from env vars - required for batch upload + const tempConfigPath = createTempRcloneConfig(accessKey, secretKey, accountId); + if (!tempConfigPath) { + throw new Error("Failed to create temporary rclone config for R2 batch upload."); } - const bucket = binding.bucket_name; - if (!bucket) { - throw new Error(`R2 binding ${JSON.stringify(R2_CACHE_BINDING_NAME)} should have a 'bucket_name'`); + const env = { + ...process.env, + RCLONE_CONFIG: tempConfigPath, + }; + + logger.info("Using batch upload with R2 credentials from environment variables"); + + // Create a staging dir in temp directory with proper key paths + const tempDir = tmpdir(); + const stagingDir = path.join(tempDir, `.r2-staging-${Date.now()}`); + + // Track success to ensure cleanup happens correctly + let success = null; + + try { + mkdirSync(stagingDir, { recursive: true }); + + for (const { fullPath, key, buildId, isFetch } of assets) { + const cacheKey = computeCacheKey(key, { + prefix, + buildId, + cacheType: isFetch ? "fetch" : "cache", + }); + const destPath = path.join(stagingDir, cacheKey); + mkdirSync(path.dirname(destPath), { recursive: true }); + copyFileSync(fullPath, destPath); + } + + // Use rclone.js to sync the R2 + const remote = `r2:${bucket}`; + + // Using rclone.js Promise-based API for the copy operation + await rclone.promises.copy(stagingDir, remote, { + progress: true, + transfers: 16, + checkers: 8, + env, + }); + + logger.info(`Successfully uploaded ${assets.length} assets to R2 using rclone batch upload`); + success = true; + } finally { + try { + // Cleanup temporary staging directory + rmSync(stagingDir, { recursive: true, force: true }); + } catch { + console.warn(`Failed to remove temporary staging directory at ${stagingDir}`); + } + + try { + // Cleanup temporary config file + rmSync(tempConfigPath); + } catch { + console.warn(`Failed to remove temporary config at ${tempConfigPath}`); + } } - const prefix = envVars[R2_CACHE_PREFIX_ENV_NAME]; + if (!success) { + throw new Error("R2 rclone batch upload failed, falling back to wrangler r2 bulk put uploads..."); + } +} - const assets = getCacheAssets(buildOpts); +/** + * Populate R2 incremental cache using Wrangler's r2 bulk put command + * Falls back to this method when batch upload is not available or fails + */ +async function populateR2WithWranglerBulkPut( + buildOpts: BuildOptions, + bucket: string, + prefix: string | undefined, + assets: CacheAsset[], + populateCacheOptions: PopulateCacheOptions, + jurisdiction?: string +) { + logger.info("Using Wrangler r2 bulk put for cache uploads."); const objectList = assets.map(({ fullPath, key, buildId, isFetch }) => ({ key: computeCacheKey(key, { @@ -240,12 +352,13 @@ async function populateR2IncrementalCache( file: fullPath, })); - const tempDir = await fsp.mkdtemp(path.join(os.tmpdir(), "open-next-")); + const tempDir = path.join(tmpdir(), `open-next-${Date.now()}`); + mkdirSync(tempDir, { recursive: true }); const listFile = path.join(tempDir, `r2-bulk-list.json`); - fs.writeFileSync(listFile, JSON.stringify(objectList)); + writeFileSync(listFile, JSON.stringify(objectList)); const concurrency = Math.max(1, populateCacheOptions.cacheChunkSize ?? 50); - const jurisdiction = binding.jurisdiction ? `--jurisdiction ${binding.jurisdiction}` : ""; + const jurisdictionFlag = jurisdiction ? `--jurisdiction ${jurisdiction}` : ""; runWrangler( buildOpts, @@ -254,7 +367,7 @@ async function populateR2IncrementalCache( bucket, `--filename ${quoteShellMeta(listFile)}`, `--concurrency ${concurrency}`, - jurisdiction, + jurisdictionFlag, ], { target: populateCacheOptions.target, @@ -266,11 +379,65 @@ async function populateR2IncrementalCache( } ); - fs.rmSync(listFile, { force: true }); + rmSync(listFile, { force: true }); logger.info(`Successfully populated cache with ${assets.length} assets`); } +async function populateR2IncrementalCache( + buildOpts: BuildOptions, + config: WranglerConfig, + populateCacheOptions: PopulateCacheOptions, + envVars: WorkerEnvVar +) { + logger.info("\nPopulating R2 incremental cache..."); + + const binding = config.r2_buckets.find(({ binding }) => binding === R2_CACHE_BINDING_NAME); + if (!binding) { + throw new Error(`No R2 binding ${JSON.stringify(R2_CACHE_BINDING_NAME)} found!`); + } + + const bucket = binding.bucket_name; + if (!bucket) { + throw new Error(`R2 binding ${JSON.stringify(R2_CACHE_BINDING_NAME)} should have a 'bucket_name'`); + } + + const prefix = envVars[R2_CACHE_PREFIX_ENV_NAME]; + + const assets = getCacheAssets(buildOpts); + + // Force sequential upload for local target + if (populateCacheOptions.target === "local") { + logger.info("Using sequential upload for local R2 (batch upload only works with remote R2)"); + return await populateR2WithWranglerBulkPut( + buildOpts, + bucket, + prefix, + assets, + populateCacheOptions, + binding.jurisdiction + ); + } + + try { + // Attempt batch upload first (using rclone) - only for remote target + return await populateR2IncrementalCacheWithBatchUpload(bucket, prefix, assets, envVars); + } catch (error) { + logger.warn(`Batch upload failed: ${error instanceof Error ? error.message : error}`); + logger.info("Falling back to sequential uploads..."); + + // Sequential upload fallback (using Wrangler) + return await populateR2WithWranglerBulkPut( + buildOpts, + bucket, + prefix, + assets, + populateCacheOptions, + binding.jurisdiction + ); + } +} + async function populateKVIncrementalCache( buildOpts: BuildOptions, config: WranglerConfig, @@ -295,7 +462,8 @@ async function populateKVIncrementalCache( logger.info(`Inserting ${assets.length} assets to KV in chunks of ${chunkSize}`); - const tempDir = await fsp.mkdtemp(path.join(os.tmpdir(), "open-next-")); + const tempDir = path.join(tmpdir(), `open-next-${Date.now()}`); + mkdirSync(tempDir, { recursive: true }); for (const i of tqdm(Array.from({ length: totalChunks }, (_, i) => i))) { const chunkPath = path.join(tempDir, `cache-chunk-${i}.json`); @@ -308,10 +476,10 @@ async function populateKVIncrementalCache( buildId, cacheType: isFetch ? "fetch" : "cache", }), - value: fs.readFileSync(fullPath, "utf8"), + value: readFileSync(fullPath, "utf8"), })); - fs.writeFileSync(chunkPath, JSON.stringify(kvMapping)); + writeFileSync(chunkPath, JSON.stringify(kvMapping)); runWrangler( buildOpts, @@ -329,7 +497,7 @@ async function populateKVIncrementalCache( } ); - fs.rmSync(chunkPath, { force: true }); + rmSync(chunkPath, { force: true }); } logger.info(`Successfully populated cache with ${assets.length} assets`); @@ -371,7 +539,7 @@ function populateD1TagCache( function populateStaticAssetsIncrementalCache(options: BuildOptions) { logger.info("\nPopulating Workers static assets..."); - fs.cpSync( + cpSync( path.join(options.outputDir, "cache"), path.join(options.outputDir, "assets", STATIC_ASSETS_CACHE_DIR), { recursive: true } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0d5d06ac3..f196cbeb0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1129,6 +1129,9 @@ importers: glob: specifier: 'catalog:' version: 12.0.0 + rclone.js: + specifier: ^0.6.6 + version: 0.6.6 ts-tqdm: specifier: ^0.8.6 version: 0.8.6 @@ -1157,6 +1160,9 @@ importers: '@types/picomatch': specifier: ^4.0.0 version: 4.0.0 + '@types/rclone.js': + specifier: ^0.6.1 + version: 0.6.3 '@types/yargs': specifier: 'catalog:' version: 17.0.33 @@ -1238,28 +1244,24 @@ packages: engines: {node: '>= 10'} cpu: [arm64] os: [linux] - libc: [glibc] '@ast-grep/napi-linux-arm64-musl@0.40.5': resolution: {integrity: sha512-/qKsmds5FMoaEj6FdNzepbmLMtlFuBLdrAn9GIWCqOIcVcYvM1Nka8+mncfeXB/MFZKOrzQsQdPTWqrrQzXLrA==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] - libc: [musl] '@ast-grep/napi-linux-x64-gnu@0.40.5': resolution: {integrity: sha512-DP4oDbq7f/1A2hRTFLhJfDFR6aI5mRWdEfKfHzRItmlKsR9WlcEl1qDJs/zX9R2EEtIDsSKRzuJNfJllY3/W8Q==} engines: {node: '>= 10'} cpu: [x64] os: [linux] - libc: [glibc] '@ast-grep/napi-linux-x64-musl@0.40.5': resolution: {integrity: sha512-BRZUvVBPUNpWPo6Ns8chXVzxHPY+k9gpsubGTHy92Q26ecZULd/dTkWWdnvfhRqttsSQ9Pe/XQdi5+hDQ6RYcg==} engines: {node: '>= 10'} cpu: [x64] os: [linux] - libc: [musl] '@ast-grep/napi-win32-arm64-msvc@0.40.5': resolution: {integrity: sha512-y95zSEwc7vhxmcrcH0GnK4ZHEBQrmrszRBNQovzaciF9GUqEcCACNLoBesn4V47IaOp4fYgD2/EhGRTIBFb2Ug==} @@ -3114,183 +3116,155 @@ packages: resolution: {integrity: sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA==} cpu: [arm64] os: [linux] - libc: [glibc] '@img/sharp-libvips-linux-arm64@1.2.4': resolution: {integrity: sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==} cpu: [arm64] os: [linux] - libc: [glibc] '@img/sharp-libvips-linux-arm@1.0.5': resolution: {integrity: sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g==} cpu: [arm] os: [linux] - libc: [glibc] '@img/sharp-libvips-linux-arm@1.2.4': resolution: {integrity: sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==} cpu: [arm] os: [linux] - libc: [glibc] '@img/sharp-libvips-linux-ppc64@1.2.4': resolution: {integrity: sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==} cpu: [ppc64] os: [linux] - libc: [glibc] '@img/sharp-libvips-linux-riscv64@1.2.4': resolution: {integrity: sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==} cpu: [riscv64] os: [linux] - libc: [glibc] '@img/sharp-libvips-linux-s390x@1.0.4': resolution: {integrity: sha512-u7Wz6ntiSSgGSGcjZ55im6uvTrOxSIS8/dgoVMoiGE9I6JAfU50yH5BoDlYA1tcuGS7g/QNtetJnxA6QEsCVTA==} cpu: [s390x] os: [linux] - libc: [glibc] '@img/sharp-libvips-linux-s390x@1.2.4': resolution: {integrity: sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==} cpu: [s390x] os: [linux] - libc: [glibc] '@img/sharp-libvips-linux-x64@1.0.4': resolution: {integrity: sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw==} cpu: [x64] os: [linux] - libc: [glibc] '@img/sharp-libvips-linux-x64@1.2.4': resolution: {integrity: sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==} cpu: [x64] os: [linux] - libc: [glibc] '@img/sharp-libvips-linuxmusl-arm64@1.0.4': resolution: {integrity: sha512-9Ti+BbTYDcsbp4wfYib8Ctm1ilkugkA/uscUn6UXK1ldpC1JjiXbLfFZtRlBhjPZ5o1NCLiDbg8fhUPKStHoTA==} cpu: [arm64] os: [linux] - libc: [musl] '@img/sharp-libvips-linuxmusl-arm64@1.2.4': resolution: {integrity: sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==} cpu: [arm64] os: [linux] - libc: [musl] '@img/sharp-libvips-linuxmusl-x64@1.0.4': resolution: {integrity: sha512-viYN1KX9m+/hGkJtvYYp+CCLgnJXwiQB39damAO7WMdKWlIhmYTfHjwSbQeUK/20vY154mwezd9HflVFM1wVSw==} cpu: [x64] os: [linux] - libc: [musl] '@img/sharp-libvips-linuxmusl-x64@1.2.4': resolution: {integrity: sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==} cpu: [x64] os: [linux] - libc: [musl] '@img/sharp-linux-arm64@0.33.5': resolution: {integrity: sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm64] os: [linux] - libc: [glibc] '@img/sharp-linux-arm64@0.34.5': resolution: {integrity: sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm64] os: [linux] - libc: [glibc] '@img/sharp-linux-arm@0.33.5': resolution: {integrity: sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm] os: [linux] - libc: [glibc] '@img/sharp-linux-arm@0.34.5': resolution: {integrity: sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm] os: [linux] - libc: [glibc] '@img/sharp-linux-ppc64@0.34.5': resolution: {integrity: sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [ppc64] os: [linux] - libc: [glibc] '@img/sharp-linux-riscv64@0.34.5': resolution: {integrity: sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [riscv64] os: [linux] - libc: [glibc] '@img/sharp-linux-s390x@0.33.5': resolution: {integrity: sha512-y/5PCd+mP4CA/sPDKl2961b+C9d+vPAveS33s6Z3zfASk2j5upL6fXVPZi7ztePZ5CuH+1kW8JtvxgbuXHRa4Q==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [s390x] os: [linux] - libc: [glibc] '@img/sharp-linux-s390x@0.34.5': resolution: {integrity: sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [s390x] os: [linux] - libc: [glibc] '@img/sharp-linux-x64@0.33.5': resolution: {integrity: sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [x64] os: [linux] - libc: [glibc] '@img/sharp-linux-x64@0.34.5': resolution: {integrity: sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [x64] os: [linux] - libc: [glibc] '@img/sharp-linuxmusl-arm64@0.33.5': resolution: {integrity: sha512-XrHMZwGQGvJg2V/oRSUfSAfjfPxO+4DkiRh6p2AFjLQztWUuY/o8Mq0eMQVIY7HJ1CDQUJlxGGZRw1a5bqmd1g==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm64] os: [linux] - libc: [musl] '@img/sharp-linuxmusl-arm64@0.34.5': resolution: {integrity: sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm64] os: [linux] - libc: [musl] '@img/sharp-linuxmusl-x64@0.33.5': resolution: {integrity: sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [x64] os: [linux] - libc: [musl] '@img/sharp-linuxmusl-x64@0.34.5': resolution: {integrity: sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [x64] os: [linux] - libc: [musl] '@img/sharp-wasm32@0.33.5': resolution: {integrity: sha512-ykUW4LVGaMcU9lu9thv85CbRMAwfeadCJHRsg2GmeRa/cJxsVY9Rbd57JcMxBkKHag5U/x7TSBpScF4U8ElVzg==} @@ -3549,140 +3523,120 @@ packages: engines: {node: '>= 10'} cpu: [arm64] os: [linux] - libc: [glibc] '@next/swc-linux-arm64-gnu@15.0.0-canary.174': resolution: {integrity: sha512-kVEibHYyQ12zzFPY+YHbYX9z81HhLVK5pQgt1NlFet2M0iBj1PxvOJuu6In1EEV7f3jNEr4r3gf5ieyY3ywnLw==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] - libc: [glibc] '@next/swc-linux-arm64-gnu@15.5.7': resolution: {integrity: sha512-NCslw3GrNIw7OgmRBxHtdWFQYhexoUCq+0oS2ccjyYLtcn1SzGzeM54jpTFonIMUjNbHmpKpziXnpxhSWLcmBA==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] - libc: [glibc] '@next/swc-linux-arm64-gnu@16.0.10': resolution: {integrity: sha512-uQtWE3X0iGB8apTIskOMi2w/MKONrPOUCi5yLO+v3O8Mb5c7K4Q5KD1jvTpTF5gJKa3VH/ijKjKUq9O9UhwOYw==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] - libc: [glibc] '@next/swc-linux-arm64-gnu@16.1.5': resolution: {integrity: sha512-qNIb42o3C02ccIeSeKjacF3HXotGsxh/FMk/rSRmCzOVMtoWH88odn2uZqF8RLsSUWHcAqTgYmPD3pZ03L9ZAA==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] - libc: [glibc] '@next/swc-linux-arm64-musl@14.2.33': resolution: {integrity: sha512-Bm+QulsAItD/x6Ih8wGIMfRJy4G73tu1HJsrccPW6AfqdZd0Sfm5Imhgkgq2+kly065rYMnCOxTBvmvFY1BKfg==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] - libc: [musl] '@next/swc-linux-arm64-musl@15.0.0-canary.174': resolution: {integrity: sha512-NzfcraJW3jpWDx3dJHzMxLFUAJxdq9GROpO49SIWXu9HKmdZszrInTfnYK98v2C73FNnpFoCGEvBYi/GTnvECw==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] - libc: [musl] '@next/swc-linux-arm64-musl@15.5.7': resolution: {integrity: sha512-nfymt+SE5cvtTrG9u1wdoxBr9bVB7mtKTcj0ltRn6gkP/2Nu1zM5ei8rwP9qKQP0Y//umK+TtkKgNtfboBxRrw==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] - libc: [musl] '@next/swc-linux-arm64-musl@16.0.10': resolution: {integrity: sha512-llA+hiDTrYvyWI21Z0L1GiXwjQaanPVQQwru5peOgtooeJ8qx3tlqRV2P7uH2pKQaUfHxI/WVarvI5oYgGxaTw==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] - libc: [musl] '@next/swc-linux-arm64-musl@16.1.5': resolution: {integrity: sha512-U+kBxGUY1xMAzDTXmuVMfhaWUZQAwzRaHJ/I6ihtR5SbTVUEaDRiEU9YMjy1obBWpdOBuk1bcm+tsmifYSygfw==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] - libc: [musl] '@next/swc-linux-x64-gnu@14.2.33': resolution: {integrity: sha512-FnFn+ZBgsVMbGDsTqo8zsnRzydvsGV8vfiWwUo1LD8FTmPTdV+otGSWKc4LJec0oSexFnCYVO4hX8P8qQKaSlg==} engines: {node: '>= 10'} cpu: [x64] os: [linux] - libc: [glibc] '@next/swc-linux-x64-gnu@15.0.0-canary.174': resolution: {integrity: sha512-fJ5W8PrbZZkxCrtX9lmlqn43zvUrQQ5wF/GxcQDFdcwT9l3lx8IhdMZH7Q5rWuikWpI0pU+jqqRdhTpODqpuHA==} engines: {node: '>= 10'} cpu: [x64] os: [linux] - libc: [glibc] '@next/swc-linux-x64-gnu@15.5.7': resolution: {integrity: sha512-hvXcZvCaaEbCZcVzcY7E1uXN9xWZfFvkNHwbe/n4OkRhFWrs1J1QV+4U1BN06tXLdaS4DazEGXwgqnu/VMcmqw==} engines: {node: '>= 10'} cpu: [x64] os: [linux] - libc: [glibc] '@next/swc-linux-x64-gnu@16.0.10': resolution: {integrity: sha512-AK2q5H0+a9nsXbeZ3FZdMtbtu9jxW4R/NgzZ6+lrTm3d6Zb7jYrWcgjcpM1k8uuqlSy4xIyPR2YiuUr+wXsavA==} engines: {node: '>= 10'} cpu: [x64] os: [linux] - libc: [glibc] '@next/swc-linux-x64-gnu@16.1.5': resolution: {integrity: sha512-gq2UtoCpN7Ke/7tKaU7i/1L7eFLfhMbXjNghSv0MVGF1dmuoaPeEVDvkDuO/9LVa44h5gqpWeJ4mRRznjDv7LA==} engines: {node: '>= 10'} cpu: [x64] os: [linux] - libc: [glibc] '@next/swc-linux-x64-musl@14.2.33': resolution: {integrity: sha512-345tsIWMzoXaQndUTDv1qypDRiebFxGYx9pYkhwY4hBRaOLt8UGfiWKr9FSSHs25dFIf8ZqIFaPdy5MljdoawA==} engines: {node: '>= 10'} cpu: [x64] os: [linux] - libc: [musl] '@next/swc-linux-x64-musl@15.0.0-canary.174': resolution: {integrity: sha512-OMSzmdZxrh5c7X46ILiK3GvTPgSZghpSFF4wrnXloBpW1LrbbjSYGVSGer5IoVqXR18lpnMscsV9N35FX0MIVw==} engines: {node: '>= 10'} cpu: [x64] os: [linux] - libc: [musl] '@next/swc-linux-x64-musl@15.5.7': resolution: {integrity: sha512-4IUO539b8FmF0odY6/SqANJdgwn1xs1GkPO5doZugwZ3ETF6JUdckk7RGmsfSf7ws8Qb2YB5It33mvNL/0acqA==} engines: {node: '>= 10'} cpu: [x64] os: [linux] - libc: [musl] '@next/swc-linux-x64-musl@16.0.10': resolution: {integrity: sha512-1TDG9PDKivNw5550S111gsO4RGennLVl9cipPhtkXIFVwo31YZ73nEbLjNC8qG3SgTz/QZyYyaFYMeY4BKZR/g==} engines: {node: '>= 10'} cpu: [x64] os: [linux] - libc: [musl] '@next/swc-linux-x64-musl@16.1.5': resolution: {integrity: sha512-bQWSE729PbXT6mMklWLf8dotislPle2L70E9q6iwETYEOt092GDn0c+TTNj26AjmeceSsC4ndyGsK5nKqHYXjQ==} engines: {node: '>= 10'} cpu: [x64] os: [linux] - libc: [musl] '@next/swc-win32-arm64-msvc@14.2.33': resolution: {integrity: sha512-nscpt0G6UCTkrT2ppnJnFsYbPDQwmum4GNXYTeoTIdsmMydSKFz9Iny2jpaRupTb+Wl298+Rh82WKzt9LCcqSQ==} @@ -3992,67 +3946,56 @@ packages: resolution: {integrity: sha512-ehSKrewwsESPt1TgSE/na9nIhWCosfGSFqv7vwEtjyAqZcvbGIg4JAcV7ZEh2tfj/IlfBeZjgOXm35iOOjadcg==} cpu: [arm] os: [linux] - libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.40.1': resolution: {integrity: sha512-m39iO/aaurh5FVIu/F4/Zsl8xppd76S4qoID8E+dSRQvTyZTOI2gVk3T4oqzfq1PtcvOfAVlwLMK3KRQMaR8lg==} cpu: [arm] os: [linux] - libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.40.1': resolution: {integrity: sha512-Y+GHnGaku4aVLSgrT0uWe2o2Rq8te9hi+MwqGF9r9ORgXhmHK5Q71N757u0F8yU1OIwUIFy6YiJtKjtyktk5hg==} cpu: [arm64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.40.1': resolution: {integrity: sha512-jEwjn3jCA+tQGswK3aEWcD09/7M5wGwc6+flhva7dsQNRZZTe30vkalgIzV4tjkopsTS9Jd7Y1Bsj6a4lzz8gQ==} cpu: [arm64] os: [linux] - libc: [musl] '@rollup/rollup-linux-loongarch64-gnu@4.40.1': resolution: {integrity: sha512-ySyWikVhNzv+BV/IDCsrraOAZ3UaC8SZB67FZlqVwXwnFhPihOso9rPOxzZbjp81suB1O2Topw+6Ug3JNegejQ==} cpu: [loong64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-powerpc64le-gnu@4.40.1': resolution: {integrity: sha512-BvvA64QxZlh7WZWqDPPdt0GH4bznuL6uOO1pmgPnnv86rpUpc8ZxgZwcEgXvo02GRIZX1hQ0j0pAnhwkhwPqWg==} cpu: [ppc64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-riscv64-gnu@4.40.1': resolution: {integrity: sha512-EQSP+8+1VuSulm9RKSMKitTav89fKbHymTf25n5+Yr6gAPZxYWpj3DzAsQqoaHAk9YX2lwEyAf9S4W8F4l3VBQ==} cpu: [riscv64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.40.1': resolution: {integrity: sha512-n/vQ4xRZXKuIpqukkMXZt9RWdl+2zgGNx7Uda8NtmLJ06NL8jiHxUawbwC+hdSq1rrw/9CghCpEONor+l1e2gA==} cpu: [riscv64] os: [linux] - libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.40.1': resolution: {integrity: sha512-h8d28xzYb98fMQKUz0w2fMc1XuGzLLjdyxVIbhbil4ELfk5/orZlSTpF/xdI9C8K0I8lCkq+1En2RJsawZekkg==} cpu: [s390x] os: [linux] - libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.40.1': resolution: {integrity: sha512-XiK5z70PEFEFqcNj3/zRSz/qX4bp4QIraTy9QjwJAb/Z8GM7kVUsD0Uk8maIPeTyPCP03ChdI+VVmJriKYbRHQ==} cpu: [x64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-x64-musl@4.40.1': resolution: {integrity: sha512-2BRORitq5rQ4Da9blVovzNCMaUlyKrzMSvkVR0D4qPuOy/+pMCrh1d7o01RATwVy+6Fa1WBw+da7QPeLWU/1mQ==} cpu: [x64] os: [linux] - libc: [musl] '@rollup/rollup-win32-arm64-msvc@4.40.1': resolution: {integrity: sha512-b2bcNm9Kbde03H+q+Jjw9tSfhYkzrDUf2d5MAd1bOJuVplXvFhWz7tRtWvD8/ORZi7qSCy0idW6tf2HgxSXQSg==} @@ -4514,28 +4457,24 @@ packages: engines: {node: '>= 10'} cpu: [arm64] os: [linux] - libc: [glibc] '@tailwindcss/oxide-linux-arm64-musl@4.1.17': resolution: {integrity: sha512-HvZLfGr42i5anKtIeQzxdkw/wPqIbpeZqe7vd3V9vI3RQxe3xU1fLjss0TjyhxWcBaipk7NYwSrwTwK1hJARMg==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] - libc: [musl] '@tailwindcss/oxide-linux-x64-gnu@4.1.17': resolution: {integrity: sha512-M3XZuORCGB7VPOEDH+nzpJ21XPvK5PyjlkSFkFziNHGLc5d6g3di2McAAblmaSUNl8IOmzYwLx9NsE7bplNkwQ==} engines: {node: '>= 10'} cpu: [x64] os: [linux] - libc: [glibc] '@tailwindcss/oxide-linux-x64-musl@4.1.17': resolution: {integrity: sha512-k7f+pf9eXLEey4pBlw+8dgfJHY4PZ5qOUFDyNf7SI6lHjQ9Zt7+NcscjpwdCEbYi6FI5c2KDTDWyf2iHcCSyyQ==} engines: {node: '>= 10'} cpu: [x64] os: [linux] - libc: [musl] '@tailwindcss/oxide-wasm32-wasi@4.1.17': resolution: {integrity: sha512-cEytGqSSoy7zK4JRWiTCx43FsKP/zGr0CsuMawhH67ONlH+T79VteQeJQRO/X7L0juEUA8ZyuYikcRBf0vsxhg==} @@ -4705,6 +4644,9 @@ packages: '@types/range-parser@1.2.7': resolution: {integrity: sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==} + '@types/rclone.js@0.6.3': + resolution: {integrity: sha512-BssKAAVRY//fxGKso8SatyOwiD7X0toDofNnVxZlIXmN7UHrn2UBTxldNAjgUvWA91qJyeEPfKmeJpZVhLugXg==} + '@types/react-dom@18.3.0': resolution: {integrity: sha512-EhwApuTmMBmXuFOikhQLIBUn6uFg81SwLMOAUgodJF14SOBOCMdU04gDoYi0WOJJHD144TL32z4yDqCW3dnkQg==} @@ -5012,6 +4954,10 @@ packages: engines: {node: '>=0.4.0'} hasBin: true + adm-zip@0.5.16: + resolution: {integrity: sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ==} + engines: {node: '>=12.0'} + agent-base@6.0.2: resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} engines: {node: '>= 6.0.0'} @@ -6663,15 +6609,18 @@ packages: glob@10.3.10: resolution: {integrity: sha512-fa46+tv1Ak0UPK1TOy/pZrIybNNt4HCv7SDzwyfiOZkvZLEbjsZkJBPtDHVshZjbecAoAGSC20MjLDG/qr679g==} engines: {node: '>=16 || 14 >=14.17'} + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me hasBin: true glob@10.4.5: resolution: {integrity: sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==} + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me hasBin: true glob@11.0.0: resolution: {integrity: sha512-9UiX/Bl6J2yaBbxKoEBRm4Cipxgok8kQYcOPEhScPwebu2I0HoQOuYdIO6S3hLuWoZgpDpwQZMzTFxgpkyT76g==} engines: {node: 20 || >=22} + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me hasBin: true glob@12.0.0: @@ -6681,11 +6630,12 @@ packages: glob@7.2.3: resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} - deprecated: Glob versions prior to v9 are no longer supported + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me glob@9.3.5: resolution: {integrity: sha512-e1LleDykUz2Iu+MTYdkSsuWX8lvAjAcs0Xef0lNIu0S2wOAzuTxCJtcd9S3cijlwYF18EsU3rzb8jPVobxDh9Q==} engines: {node: '>=16 || 14 >=14.17'} + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me globals@13.24.0: resolution: {integrity: sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==} @@ -7311,28 +7261,24 @@ packages: engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] - libc: [glibc] lightningcss-linux-arm64-musl@1.30.2: resolution: {integrity: sha512-5Vh9dGeblpTxWHpOx8iauV02popZDsCYMPIgiuw97OJ5uaDsL86cnqSFs5LZkG3ghHoX5isLgWzMs+eD1YzrnA==} engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] - libc: [musl] lightningcss-linux-x64-gnu@1.30.2: resolution: {integrity: sha512-Cfd46gdmj1vQ+lR6VRTTadNHu6ALuw2pKR9lYq4FnhvgBc4zWY1EtZcAc6EffShbb1MFrIPfLDXD6Xprbnni4w==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] - libc: [glibc] lightningcss-linux-x64-musl@1.30.2: resolution: {integrity: sha512-XJaLUUFXb6/QG2lGIW6aIk6jKdtjtcffUT0NKvIqhSBY3hh9Ch+1LCeH80dR9q9LBjG3ewbDjnumefsLsP6aiA==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] - libc: [musl] lightningcss-win32-arm64-msvc@1.30.2: resolution: {integrity: sha512-FZn+vaj7zLv//D/192WFFVA0RgHawIcHqLX9xuWiQt7P0PtdFEVaxgF9rjM/IRYHQXNnk61/H/gb2Ei+kUQ4xQ==} @@ -8475,6 +8421,13 @@ packages: resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} hasBin: true + rclone.js@0.6.6: + resolution: {integrity: sha512-Dxh34cab/fNjFq5SSm0fYLNkGzG2cQSBy782UW9WwxJCEiVO4cGXkvaXcNlgv817dK8K8PuQ+NHUqSAMMhWujQ==} + engines: {node: '>=12'} + cpu: [arm, arm64, mips, mipsel, x32, x64] + os: [darwin, freebsd, linux, openbsd, sunos, win32] + hasBin: true + react-dom@18.3.1: resolution: {integrity: sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==} peerDependencies: @@ -9112,10 +9065,12 @@ packages: tar@4.4.18: resolution: {integrity: sha512-ZuOtqqmkV9RE1+4odd+MhBpibmCxNP6PJhH/h2OqNuotTX7/XHPZQJv2pKvWMplFH9SIZZhitehh6vBH6LO8Pg==} engines: {node: '>=4.5'} + deprecated: Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me tar@7.4.3: resolution: {integrity: sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==} engines: {node: '>=18'} + deprecated: Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me tarn@3.0.2: resolution: {integrity: sha512-51LAVKUSZSVfI05vjPESNc5vwqqZpbXCsU+/+wxlOrUjk2SnFTt97v9ZgQrD4YmxYW1Px6w2KjaDitCfkvgxMQ==} @@ -13734,6 +13689,10 @@ snapshots: '@types/range-parser@1.2.7': {} + '@types/rclone.js@0.6.3': + dependencies: + '@types/node': 20.14.10 + '@types/react-dom@18.3.0': dependencies: '@types/react': 19.0.8 @@ -14398,6 +14357,8 @@ snapshots: acorn@8.15.0: {} + adm-zip@0.5.16: {} + agent-base@6.0.2: dependencies: debug: 4.4.0 @@ -15578,8 +15539,8 @@ snapshots: '@typescript-eslint/parser': 8.7.0(eslint@8.57.1)(typescript@5.9.3) eslint: 8.57.1 eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1))(eslint@8.57.1) - eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1) + eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0)(eslint@8.57.1) + eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1) eslint-plugin-jsx-a11y: 6.10.0(eslint@8.57.1) eslint-plugin-react: 7.36.1(eslint@8.57.1) eslint-plugin-react-hooks: 4.6.2(eslint@8.57.1) @@ -15598,8 +15559,8 @@ snapshots: '@typescript-eslint/parser': 8.7.0(eslint@8.57.1)(typescript@5.7.3) eslint: 8.57.1 eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint@8.57.1))(eslint@8.57.1) - eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1) + eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0)(eslint@8.57.1) + eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint@8.57.1) eslint-plugin-jsx-a11y: 6.10.0(eslint@8.57.1) eslint-plugin-react: 7.36.1(eslint@8.57.1) eslint-plugin-react-hooks: 5.1.0(eslint@8.57.1) @@ -15638,8 +15599,8 @@ snapshots: '@typescript-eslint/parser': 8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3) eslint: 9.19.0(jiti@2.6.1) eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint@9.19.0(jiti@2.6.1)))(eslint@9.19.0(jiti@2.6.1)) - eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint-import-resolver-typescript@3.6.3)(eslint@9.19.0(jiti@2.6.1)) + eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0)(eslint@9.19.0(jiti@2.6.1)) + eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint@9.19.0(jiti@2.6.1)) eslint-plugin-jsx-a11y: 6.10.0(eslint@9.19.0(jiti@2.6.1)) eslint-plugin-react: 7.37.4(eslint@9.19.0(jiti@2.6.1)) eslint-plugin-react-hooks: 5.1.0(eslint@9.19.0(jiti@2.6.1)) @@ -15658,38 +15619,38 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint@8.57.1))(eslint@8.57.1): + eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0)(eslint@8.57.1): dependencies: '@nolyfill/is-core-module': 1.0.39 debug: 4.4.0 enhanced-resolve: 5.17.1 eslint: 8.57.1 - eslint-module-utils: 2.11.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1) + eslint-module-utils: 2.11.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1) fast-glob: 3.3.2 get-tsconfig: 4.8.0 is-bun-module: 1.2.1 is-glob: 4.0.3 optionalDependencies: - eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1) + eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint@8.57.1) transitivePeerDependencies: - '@typescript-eslint/parser' - eslint-import-resolver-node - eslint-import-resolver-webpack - supports-color - eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1))(eslint@8.57.1): + eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0)(eslint@8.57.1): dependencies: '@nolyfill/is-core-module': 1.0.39 debug: 4.4.0 enhanced-resolve: 5.17.1 eslint: 8.57.1 - eslint-module-utils: 2.11.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1) + eslint-module-utils: 2.11.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1) fast-glob: 3.3.2 get-tsconfig: 4.8.0 is-bun-module: 1.2.1 is-glob: 4.0.3 optionalDependencies: - eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1) + eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1) transitivePeerDependencies: - '@typescript-eslint/parser' - eslint-import-resolver-node @@ -15715,44 +15676,44 @@ snapshots: - eslint-import-resolver-webpack - supports-color - eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint@9.19.0(jiti@2.6.1)))(eslint@9.19.0(jiti@2.6.1)): + eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0)(eslint@9.19.0(jiti@2.6.1)): dependencies: '@nolyfill/is-core-module': 1.0.39 debug: 4.4.0 enhanced-resolve: 5.17.1 eslint: 9.19.0(jiti@2.6.1) - eslint-module-utils: 2.11.0(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint@9.19.0(jiti@2.6.1)))(eslint@9.19.0(jiti@2.6.1)))(eslint@9.19.0(jiti@2.6.1)) + eslint-module-utils: 2.11.0(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3)(eslint@9.19.0(jiti@2.6.1)) fast-glob: 3.3.2 get-tsconfig: 4.8.0 is-bun-module: 1.2.1 is-glob: 4.0.3 optionalDependencies: - eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint-import-resolver-typescript@3.6.3)(eslint@9.19.0(jiti@2.6.1)) + eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint@9.19.0(jiti@2.6.1)) transitivePeerDependencies: - '@typescript-eslint/parser' - eslint-import-resolver-node - eslint-import-resolver-webpack - supports-color - eslint-module-utils@2.11.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1): + eslint-module-utils@2.11.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1): dependencies: debug: 3.2.7 optionalDependencies: '@typescript-eslint/parser': 8.7.0(eslint@8.57.1)(typescript@5.7.3) eslint: 8.57.1 eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint@8.57.1))(eslint@8.57.1) + eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0)(eslint@8.57.1) transitivePeerDependencies: - supports-color - eslint-module-utils@2.11.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1): + eslint-module-utils@2.11.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1): dependencies: debug: 3.2.7 optionalDependencies: '@typescript-eslint/parser': 8.7.0(eslint@8.57.1)(typescript@5.9.3) eslint: 8.57.1 eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1))(eslint@8.57.1) + eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0)(eslint@8.57.1) transitivePeerDependencies: - supports-color @@ -15767,14 +15728,14 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-module-utils@2.11.0(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint@9.19.0(jiti@2.6.1)))(eslint@9.19.0(jiti@2.6.1)))(eslint@9.19.0(jiti@2.6.1)): + eslint-module-utils@2.11.0(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3)(eslint@9.19.0(jiti@2.6.1)): dependencies: debug: 3.2.7 optionalDependencies: '@typescript-eslint/parser': 8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3) eslint: 9.19.0(jiti@2.6.1) eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint@9.19.0(jiti@2.6.1)))(eslint@9.19.0(jiti@2.6.1)) + eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0)(eslint@9.19.0(jiti@2.6.1)) transitivePeerDependencies: - supports-color @@ -15788,25 +15749,25 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-module-utils@2.12.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1): + eslint-module-utils@2.12.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1): dependencies: debug: 3.2.7 optionalDependencies: '@typescript-eslint/parser': 8.7.0(eslint@8.57.1)(typescript@5.7.3) eslint: 8.57.1 eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint@8.57.1))(eslint@8.57.1) + eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0)(eslint@8.57.1) transitivePeerDependencies: - supports-color - eslint-module-utils@2.12.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1): + eslint-module-utils@2.12.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1): dependencies: debug: 3.2.7 optionalDependencies: '@typescript-eslint/parser': 8.7.0(eslint@8.57.1)(typescript@5.9.3) eslint: 8.57.1 eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1))(eslint@8.57.1) + eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0)(eslint@8.57.1) transitivePeerDependencies: - supports-color @@ -15821,14 +15782,14 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-module-utils@2.12.0(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint@9.19.0(jiti@2.6.1)))(eslint@9.19.0(jiti@2.6.1)))(eslint@9.19.0(jiti@2.6.1)): + eslint-module-utils@2.12.0(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3)(eslint@9.19.0(jiti@2.6.1)): dependencies: debug: 3.2.7 optionalDependencies: '@typescript-eslint/parser': 8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3) eslint: 9.19.0(jiti@2.6.1) eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint@9.19.0(jiti@2.6.1)))(eslint@9.19.0(jiti@2.6.1)) + eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0)(eslint@9.19.0(jiti@2.6.1)) transitivePeerDependencies: - supports-color @@ -15861,7 +15822,7 @@ snapshots: - eslint-import-resolver-webpack - supports-color - eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1): + eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint@8.57.1): dependencies: '@rtsao/scc': 1.1.0 array-includes: 3.1.8 @@ -15872,7 +15833,7 @@ snapshots: doctrine: 2.1.0 eslint: 8.57.1 eslint-import-resolver-node: 0.3.9 - eslint-module-utils: 2.12.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1) + eslint-module-utils: 2.12.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1) hasown: 2.0.2 is-core-module: 2.16.1 is-glob: 4.0.3 @@ -15890,7 +15851,7 @@ snapshots: - eslint-import-resolver-webpack - supports-color - eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1): + eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1): dependencies: '@rtsao/scc': 1.1.0 array-includes: 3.1.8 @@ -15901,7 +15862,7 @@ snapshots: doctrine: 2.1.0 eslint: 8.57.1 eslint-import-resolver-node: 0.3.9 - eslint-module-utils: 2.12.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1))(eslint@8.57.1))(eslint@8.57.1) + eslint-module-utils: 2.12.0(@typescript-eslint/parser@8.7.0(eslint@8.57.1)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1) hasown: 2.0.2 is-core-module: 2.16.1 is-glob: 4.0.3 @@ -15948,7 +15909,7 @@ snapshots: - eslint-import-resolver-webpack - supports-color - eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint-import-resolver-typescript@3.6.3)(eslint@9.19.0(jiti@2.6.1)): + eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint@9.19.0(jiti@2.6.1)): dependencies: '@rtsao/scc': 1.1.0 array-includes: 3.1.8 @@ -15959,7 +15920,7 @@ snapshots: doctrine: 2.1.0 eslint: 9.19.0(jiti@2.6.1) eslint-import-resolver-node: 0.3.9 - eslint-module-utils: 2.12.0(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint@9.19.0(jiti@2.6.1)))(eslint@9.19.0(jiti@2.6.1)))(eslint@9.19.0(jiti@2.6.1)) + eslint-module-utils: 2.12.0(@typescript-eslint/parser@8.7.0(eslint@9.19.0(jiti@2.6.1))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3)(eslint@9.19.0(jiti@2.6.1)) hasown: 2.0.2 is-core-module: 2.16.1 is-glob: 4.0.3 @@ -18852,6 +18813,11 @@ snapshots: minimist: 1.2.8 strip-json-comments: 2.0.1 + rclone.js@0.6.6: + dependencies: + adm-zip: 0.5.16 + mri: 1.2.0 + react-dom@18.3.1(react@18.3.1): dependencies: loose-envify: 1.4.0 From 89293a63aa555f6f74a500348a4bf42d3d3bd7e0 Mon Sep 17 00:00:00 2001 From: Mateusz Grab Date: Fri, 6 Feb 2026 10:46:21 +0100 Subject: [PATCH 02/10] Update Changeset to reflect changes against current main branch --- .changeset/recovered-rclone-batch-upload.md | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/.changeset/recovered-rclone-batch-upload.md b/.changeset/recovered-rclone-batch-upload.md index d15e9d315..7fe2a3857 100644 --- a/.changeset/recovered-rclone-batch-upload.md +++ b/.changeset/recovered-rclone-batch-upload.md @@ -2,34 +2,27 @@ "@opennextjs/cloudflare": minor --- -feature: optional batch upload for faster R2 cache population +feature: optional batch upload via rclone for fast R2 cache population that bypasses Account Level Rate Limits -This update recovers optional batch upload support for R2 cache population, significantly improving upload performance for large caches when enabled via .env or environment variables. +This update recovers optional opt in for batch upload support for R2 cache population via rclone, which bypasses Account Level Rate Limits. **Key Changes:** -1. **Optional Batch Upload**: Configure R2 credentials via .env or environment variables to enable faster batch uploads: +1. **Optional Batch Upload**: Configure R2 credentials via .env or environment variables to opt in to rclone based batch uploads: - `R2_ACCESS_KEY_ID` - `R2_SECRET_ACCESS_KEY` - `CF_ACCOUNT_ID` -2. **Automatic Detection**: When credentials are detected, batch upload is automatically used for better performance +2. **Automatic Detection**: When credentials are detected, batch upload is automatically used -3. **Smart Fallback**: If credentials are not configured, the CLI falls back to standard Wrangler uploads with a helpful message about enabling batch upload for better performance +3. **Smart Fallback**: If credentials are not configured, the CLI falls back to standard Wrangler uploads with a helpful message about enabling batch upload to bypass Account Level Rate Limits -**All deployment commands support batch upload:** - -- `populateCache` - Explicit cache population -- `deploy` - Deploy with cache population -- `upload` - Upload version with cache population -- `preview` - Preview with cache population - -**Performance Benefits (when batch upload is enabled):** +**Benefits (when batch upload is enabled):** - Parallel transfer capabilities (32 concurrent transfers) -- Significantly faster for large caches - Reduced API calls to Cloudflare +- Bypassing Account Level Rate Limits **Usage:** From f861f310586cf902b07e25c14e16f4005f6c9112 Mon Sep 17 00:00:00 2001 From: Mateusz Grab Date: Fri, 6 Feb 2026 10:51:36 +0100 Subject: [PATCH 03/10] Fix formatting --- packages/cloudflare/package.json | 2 +- packages/cloudflare/src/api/cloudflare-context.ts | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/cloudflare/package.json b/packages/cloudflare/package.json index 271dfa9d0..ba0c2cb39 100644 --- a/packages/cloudflare/package.json +++ b/packages/cloudflare/package.json @@ -90,4 +90,4 @@ "wrangler": "catalog:", "next": "~15.0.8 || ~15.1.12 || ~15.2.9 || ~15.3.9 || ~15.4.11 || ~15.5.10 || ~16.0.11 || ^16.1.5" } -} \ No newline at end of file +} diff --git a/packages/cloudflare/src/api/cloudflare-context.ts b/packages/cloudflare/src/api/cloudflare-context.ts index 2975b3a33..8400ed704 100644 --- a/packages/cloudflare/src/api/cloudflare-context.ts +++ b/packages/cloudflare/src/api/cloudflare-context.ts @@ -205,10 +205,10 @@ function getCloudflareContextSync< if (inSSG()) { throw new Error( `\n\nERROR: \`getCloudflareContext\` has been called in sync mode in either a static route or at the top level of a non-static one,` + - ` both cases are not allowed but can be solved by either:\n` + - ` - make sure that the call is not at the top level and that the route is not static\n` + - ` - call \`getCloudflareContext({async: true})\` to use the \`async\` mode\n` + - ` - avoid calling \`getCloudflareContext\` in the route\n` + ` both cases are not allowed but can be solved by either:\n` + + ` - make sure that the call is not at the top level and that the route is not static\n` + + ` - call \`getCloudflareContext({async: true})\` to use the \`async\` mode\n` + + ` - avoid calling \`getCloudflareContext\` in the route\n` ); } @@ -257,7 +257,7 @@ export async function initOpenNextCloudflareForDev(options?: GetPlatformProxyOpt if (options?.environment && process.env.NEXT_DEV_WRANGLER_ENV) { console.warn( `'initOpenNextCloudflareForDev' has been called with an environment option while NEXT_DEV_WRANGLER_ENV is set.` + - ` NEXT_DEV_WRANGLER_ENV will be ignored and the environment will be set to: '${options.environment}'` + ` NEXT_DEV_WRANGLER_ENV will be ignored and the environment will be set to: '${options.environment}'` ); } From 4f6a0ab5dd12034831a78defe8ce7377a0c70c70 Mon Sep 17 00:00:00 2001 From: Mateusz Grab Date: Fri, 6 Feb 2026 10:53:47 +0100 Subject: [PATCH 04/10] Fix comment --- packages/cloudflare/src/cli/commands/populate-cache.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cloudflare/src/cli/commands/populate-cache.ts b/packages/cloudflare/src/cli/commands/populate-cache.ts index d9e7dab81..ea21d84ec 100644 --- a/packages/cloudflare/src/cli/commands/populate-cache.ts +++ b/packages/cloudflare/src/cli/commands/populate-cache.ts @@ -406,7 +406,7 @@ async function populateR2IncrementalCache( const assets = getCacheAssets(buildOpts); - // Force sequential upload for local target + // Force wrangler r2 bulk upload for local target if (populateCacheOptions.target === "local") { logger.info("Using sequential upload for local R2 (batch upload only works with remote R2)"); return await populateR2WithWranglerBulkPut( From 9004a4a3ffb9374f62b9929c1ca3ec3cbef5462b Mon Sep 17 00:00:00 2001 From: Mateusz Grab Date: Fri, 6 Feb 2026 11:11:41 +0100 Subject: [PATCH 05/10] Fix all remarks to PR --- .changeset/recovered-rclone-batch-upload.md | 28 +++++++++++++------ .../src/cli/commands/populate-cache.ts | 16 +++++------ 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/.changeset/recovered-rclone-batch-upload.md b/.changeset/recovered-rclone-batch-upload.md index 7fe2a3857..1699e46b5 100644 --- a/.changeset/recovered-rclone-batch-upload.md +++ b/.changeset/recovered-rclone-batch-upload.md @@ -2,27 +2,37 @@ "@opennextjs/cloudflare": minor --- -feature: optional batch upload via rclone for fast R2 cache population that bypasses Account Level Rate Limits +feature: optional rclone batch upload for faster R2 cache population -This update recovers optional opt in for batch upload support for R2 cache population via rclone, which bypasses Account Level Rate Limits. +This update recovers optional rclone batch upload support for R2 cache population, significantly improving upload performance for large caches. This is an opt-in feature that requires explicit configuration via environment variables. + +**Important:** rclone doesn't work on all platforms and only works with remote deployments (not in dev/preview mode). **Key Changes:** -1. **Optional Batch Upload**: Configure R2 credentials via .env or environment variables to opt in to rclone based batch uploads: +1. **Opt-in rclone Batch Upload**: Configure R2 credentials via .env or environment variables to enable faster parallel uploads using rclone: - `R2_ACCESS_KEY_ID` - `R2_SECRET_ACCESS_KEY` - `CF_ACCOUNT_ID` -2. **Automatic Detection**: When credentials are detected, batch upload is automatically used +2. **Automatic Detection**: When credentials are detected and target is remote, rclone batch upload is automatically used for better performance + +3. **Smart Fallback**: If credentials are not configured or rclone fails, the CLI falls back to standard Wrangler r2 bulk put with a helpful message + +**Deployment commands that support rclone batch upload (remote target only):** + +- `populateCache remote` - Explicit cache population for remote +- `deploy` - Deploy with cache population +- `upload` - Upload version with cache population -3. **Smart Fallback**: If credentials are not configured, the CLI falls back to standard Wrangler uploads with a helpful message about enabling batch upload to bypass Account Level Rate Limits +Note: Batch upload does not work with local target or in dev/preview modes. -**Benefits (when batch upload is enabled):** +**Benefits (when rclone batch upload is enabled):** -- Parallel transfer capabilities (32 concurrent transfers) -- Reduced API calls to Cloudflare -- Bypassing Account Level Rate Limits +- Parallel transfer capabilities (16 concurrent transfers with 8 checkers) +- Significantly faster for large caches +- Bypasses Account Rate Limits **Usage:** diff --git a/packages/cloudflare/src/cli/commands/populate-cache.ts b/packages/cloudflare/src/cli/commands/populate-cache.ts index ea21d84ec..f0ad2a133 100644 --- a/packages/cloudflare/src/cli/commands/populate-cache.ts +++ b/packages/cloudflare/src/cli/commands/populate-cache.ts @@ -1,4 +1,5 @@ import { copyFileSync, cpSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import fsp from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; @@ -239,8 +240,8 @@ acl = private } /** - * Populate R2 incremental cache using batch upload for better performance - * Uses parallel transfers to significantly speed up cache population + * Populate R2 incremental cache using rclone for batch upload + * Uses rclone with parallel transfers to significantly speed up cache population */ async function populateR2IncrementalCacheWithBatchUpload( bucket: string, @@ -406,9 +407,9 @@ async function populateR2IncrementalCache( const assets = getCacheAssets(buildOpts); - // Force wrangler r2 bulk upload for local target + // Force wrangler r2 bulk put for local target if (populateCacheOptions.target === "local") { - logger.info("Using sequential upload for local R2 (batch upload only works with remote R2)"); + logger.info("Using wrangler r2 bulk put for local R2 (rclone batch upload only works with remote R2)"); return await populateR2WithWranglerBulkPut( buildOpts, bucket, @@ -424,9 +425,9 @@ async function populateR2IncrementalCache( return await populateR2IncrementalCacheWithBatchUpload(bucket, prefix, assets, envVars); } catch (error) { logger.warn(`Batch upload failed: ${error instanceof Error ? error.message : error}`); - logger.info("Falling back to sequential uploads..."); + logger.info("Falling back to wrangler r2 bulk put..."); - // Sequential upload fallback (using Wrangler) + // Wrangler r2 bulk put fallback return await populateR2WithWranglerBulkPut( buildOpts, bucket, @@ -462,8 +463,7 @@ async function populateKVIncrementalCache( logger.info(`Inserting ${assets.length} assets to KV in chunks of ${chunkSize}`); - const tempDir = path.join(tmpdir(), `open-next-${Date.now()}`); - mkdirSync(tempDir, { recursive: true }); + const tempDir = await fsp.mkdtemp(path.join(tmpdir(), "open-next-")); for (const i of tqdm(Array.from({ length: totalChunks }, (_, i) => i))) { const chunkPath = path.join(tempDir, `cache-chunk-${i}.json`); From 174547c5c2a548b7e34ab58dbebce4838974b7e2 Mon Sep 17 00:00:00 2001 From: Mateusz Grab Date: Fri, 6 Feb 2026 11:50:22 +0100 Subject: [PATCH 06/10] changeset updated --- .changeset/recovered-rclone-batch-upload.md | 34 ++++++++------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/.changeset/recovered-rclone-batch-upload.md b/.changeset/recovered-rclone-batch-upload.md index 1699e46b5..bc84d2efc 100644 --- a/.changeset/recovered-rclone-batch-upload.md +++ b/.changeset/recovered-rclone-batch-upload.md @@ -2,37 +2,27 @@ "@opennextjs/cloudflare": minor --- -feature: optional rclone batch upload for faster R2 cache population +feature: optional batch upload via rclone for fast R2 cache population that bypasses Account Level Rate Limits -This update recovers optional rclone batch upload support for R2 cache population, significantly improving upload performance for large caches. This is an opt-in feature that requires explicit configuration via environment variables. - -**Important:** rclone doesn't work on all platforms and only works with remote deployments (not in dev/preview mode). +This update recovers optional opt in for batch upload support for R2 cache population via rclone, which bypasses Account Level Rate Limits. **Key Changes:** -1. **Opt-in rclone Batch Upload**: Configure R2 credentials via .env or environment variables to enable faster parallel uploads using rclone: - - - `R2_ACCESS_KEY_ID` - - `R2_SECRET_ACCESS_KEY` - - `CF_ACCOUNT_ID` - -2. **Automatic Detection**: When credentials are detected and target is remote, rclone batch upload is automatically used for better performance - -3. **Smart Fallback**: If credentials are not configured or rclone fails, the CLI falls back to standard Wrangler r2 bulk put with a helpful message +1. **Optional Batch Upload**: Configure R2 credentials via .env or environment variables to opt in to rclone based batch uploads: -**Deployment commands that support rclone batch upload (remote target only):** + - `R2_ACCESS_KEY_ID` + - `R2_SECRET_ACCESS_KEY` + - `CF_ACCOUNT_ID` -- `populateCache remote` - Explicit cache population for remote -- `deploy` - Deploy with cache population -- `upload` - Upload version with cache population +2. **Automatic Detection**: When credentials are detected, batch upload is automatically used -Note: Batch upload does not work with local target or in dev/preview modes. +3. **Smart Fallback**: If credentials are not configured, the CLI falls back to standard Wrangler uploads with a helpful message about enabling batch upload to bypass Account Level Rate Limits -**Benefits (when rclone batch upload is enabled):** +**Benefits (when batch upload is enabled):** -- Parallel transfer capabilities (16 concurrent transfers with 8 checkers) -- Significantly faster for large caches -- Bypasses Account Rate Limits +- Parallel transfer capabilities (32 concurrent transfers) +- Reduced API calls to Cloudflare +- Bypassing Account Level Rate Limits **Usage:** From 58f6d404e0d63a20b01792029a43fc6f03ef5752 Mon Sep 17 00:00:00 2001 From: grabmateusz Date: Fri, 6 Feb 2026 14:25:12 +0100 Subject: [PATCH 07/10] Apply suggestion from @vicb Co-authored-by: Victor Berchet --- .changeset/recovered-rclone-batch-upload.md | 6 ++--- .../src/cli/commands/populate-cache.ts | 25 ++++++------------- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/.changeset/recovered-rclone-batch-upload.md b/.changeset/recovered-rclone-batch-upload.md index bc84d2efc..afa709dcc 100644 --- a/.changeset/recovered-rclone-batch-upload.md +++ b/.changeset/recovered-rclone-batch-upload.md @@ -2,13 +2,13 @@ "@opennextjs/cloudflare": minor --- -feature: optional batch upload via rclone for fast R2 cache population that bypasses Account Level Rate Limits +feature: optional batch upload via rclone for fast R2 cache population. -This update recovers optional opt in for batch upload support for R2 cache population via rclone, which bypasses Account Level Rate Limits. +This PR implement batch upload support for R2 cache population via rclone, which is not subject to the R2 REST API limits. **Key Changes:** -1. **Optional Batch Upload**: Configure R2 credentials via .env or environment variables to opt in to rclone based batch uploads: +1. **Optional Batch Upload**: Configure R2 credentials via `.env` or secrets to opt in to rclone based batch uploads: - `R2_ACCESS_KEY_ID` - `R2_SECRET_ACCESS_KEY` diff --git a/packages/cloudflare/src/cli/commands/populate-cache.ts b/packages/cloudflare/src/cli/commands/populate-cache.ts index f0ad2a133..805ac6402 100644 --- a/packages/cloudflare/src/cli/commands/populate-cache.ts +++ b/packages/cloudflare/src/cli/commands/populate-cache.ts @@ -209,7 +209,7 @@ type PopulateCacheOptions = { }; /** - * Create a temporary configuration file for batch upload from environment variables + * Create a temporary rclone configuration file. * @returns Path to the temporary config file or null if env vars not available */ function createTempRcloneConfig(accessKey: string, secretKey: string, accountId: string): string | null { @@ -225,16 +225,7 @@ endpoint = https://${accountId}.r2.cloudflarestorage.com acl = private `; - /** - * 0o600 is an octal number (the 0o prefix indicates octal in JavaScript) - * that represents Unix file permissions: - * - * - 6 (owner): read (4) + write (2) = readable and writable by the file owner - * - 0 (group): no permissions for the group - * - 0 (others): no permissions for anyone else - * - * In symbolic notation, this is: rw------- - */ + // 0o600 means readable and writable by the file owner writeFileSync(tempConfigPath, configContent, { mode: 0o600 }); return tempConfigPath; } @@ -260,7 +251,7 @@ async function populateR2IncrementalCacheWithBatchUpload( ); } - logger.info("\nPopulating remote R2 incremental cache using batch upload..."); + logger.info("\nPopulating remote R2 incremental cache using rclone..."); // Create temporary config from env vars - required for batch upload const tempConfigPath = createTempRcloneConfig(accessKey, secretKey, accountId); @@ -273,7 +264,7 @@ async function populateR2IncrementalCacheWithBatchUpload( RCLONE_CONFIG: tempConfigPath, }; - logger.info("Using batch upload with R2 credentials from environment variables"); + logger.info("Using rclone with R2 credentials from environment variables"); // Create a staging dir in temp directory with proper key paths const tempDir = tmpdir(); @@ -307,14 +298,14 @@ async function populateR2IncrementalCacheWithBatchUpload( env, }); - logger.info(`Successfully uploaded ${assets.length} assets to R2 using rclone batch upload`); + logger.info(`Successfully uploaded ${assets.length} assets to R2 using rclone`); success = true; } finally { try { // Cleanup temporary staging directory rmSync(stagingDir, { recursive: true, force: true }); } catch { - console.warn(`Failed to remove temporary staging directory at ${stagingDir}`); + logger.info(`Failed to remove temporary staging directory at ${stagingDir}`); } try { @@ -326,13 +317,13 @@ async function populateR2IncrementalCacheWithBatchUpload( } if (!success) { - throw new Error("R2 rclone batch upload failed, falling back to wrangler r2 bulk put uploads..."); + throw new Error("R2 rclone failed, falling back to wrangler r2 bulk put uploads..."); } } /** * Populate R2 incremental cache using Wrangler's r2 bulk put command - * Falls back to this method when batch upload is not available or fails + * Falls back to this method when rclone is not available or fails */ async function populateR2WithWranglerBulkPut( buildOpts: BuildOptions, From 4e82fe201546ca88c8757dda556d68c3e464196d Mon Sep 17 00:00:00 2001 From: Mateusz Grab Date: Fri, 6 Feb 2026 14:35:59 +0100 Subject: [PATCH 08/10] Fix remarks to PR --- .../src/cli/commands/populate-cache.ts | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/packages/cloudflare/src/cli/commands/populate-cache.ts b/packages/cloudflare/src/cli/commands/populate-cache.ts index 805ac6402..a126f78a7 100644 --- a/packages/cloudflare/src/cli/commands/populate-cache.ts +++ b/packages/cloudflare/src/cli/commands/populate-cache.ts @@ -1,6 +1,6 @@ -import { copyFileSync, cpSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import fs from "node:fs"; import fsp from "node:fs/promises"; -import { tmpdir } from "node:os"; +import os from "node:os"; import path from "node:path"; import type { BuildOptions } from "@opennextjs/aws/build/helper.js"; @@ -92,7 +92,7 @@ export async function populateCache( ) { const { incrementalCache, tagCache } = config.default.override ?? {}; - if (!existsSync(buildOpts.outputDir)) { + if (!fs.existsSync(buildOpts.outputDir)) { logger.error("Unable to populate cache: Open Next build not found"); process.exit(1); } @@ -213,7 +213,7 @@ type PopulateCacheOptions = { * @returns Path to the temporary config file or null if env vars not available */ function createTempRcloneConfig(accessKey: string, secretKey: string, accountId: string): string | null { - const tempDir = tmpdir(); + const tempDir = os.tmpdir(); const tempConfigPath = path.join(tempDir, `rclone-config-${Date.now()}.conf`); const configContent = `[r2] @@ -226,7 +226,7 @@ acl = private `; // 0o600 means readable and writable by the file owner - writeFileSync(tempConfigPath, configContent, { mode: 0o600 }); + fs.writeFileSync(tempConfigPath, configContent, { mode: 0o600 }); return tempConfigPath; } @@ -267,14 +267,14 @@ async function populateR2IncrementalCacheWithBatchUpload( logger.info("Using rclone with R2 credentials from environment variables"); // Create a staging dir in temp directory with proper key paths - const tempDir = tmpdir(); + const tempDir = os.tmpdir(); const stagingDir = path.join(tempDir, `.r2-staging-${Date.now()}`); // Track success to ensure cleanup happens correctly let success = null; try { - mkdirSync(stagingDir, { recursive: true }); + fs.mkdirSync(stagingDir, { recursive: true }); for (const { fullPath, key, buildId, isFetch } of assets) { const cacheKey = computeCacheKey(key, { @@ -283,8 +283,8 @@ async function populateR2IncrementalCacheWithBatchUpload( cacheType: isFetch ? "fetch" : "cache", }); const destPath = path.join(stagingDir, cacheKey); - mkdirSync(path.dirname(destPath), { recursive: true }); - copyFileSync(fullPath, destPath); + fs.mkdirSync(path.dirname(destPath), { recursive: true }); + fs.copyFileSync(fullPath, destPath); } // Use rclone.js to sync the R2 @@ -303,16 +303,16 @@ async function populateR2IncrementalCacheWithBatchUpload( } finally { try { // Cleanup temporary staging directory - rmSync(stagingDir, { recursive: true, force: true }); + fs.rmSync(stagingDir, { recursive: true, force: true }); } catch { logger.info(`Failed to remove temporary staging directory at ${stagingDir}`); } try { // Cleanup temporary config file - rmSync(tempConfigPath); + fs.rmSync(tempConfigPath); } catch { - console.warn(`Failed to remove temporary config at ${tempConfigPath}`); + logger.warn(`Failed to remove temporary config at ${tempConfigPath}`); } } @@ -344,10 +344,10 @@ async function populateR2WithWranglerBulkPut( file: fullPath, })); - const tempDir = path.join(tmpdir(), `open-next-${Date.now()}`); - mkdirSync(tempDir, { recursive: true }); + const tempDir = path.join(os.tmpdir(), `open-next-${Date.now()}`); + fs.mkdirSync(tempDir, { recursive: true }); const listFile = path.join(tempDir, `r2-bulk-list.json`); - writeFileSync(listFile, JSON.stringify(objectList)); + fs.writeFileSync(listFile, JSON.stringify(objectList)); const concurrency = Math.max(1, populateCacheOptions.cacheChunkSize ?? 50); const jurisdictionFlag = jurisdiction ? `--jurisdiction ${jurisdiction}` : ""; @@ -371,7 +371,7 @@ async function populateR2WithWranglerBulkPut( } ); - rmSync(listFile, { force: true }); + fs.rmSync(listFile, { force: true }); logger.info(`Successfully populated cache with ${assets.length} assets`); } @@ -454,7 +454,7 @@ async function populateKVIncrementalCache( logger.info(`Inserting ${assets.length} assets to KV in chunks of ${chunkSize}`); - const tempDir = await fsp.mkdtemp(path.join(tmpdir(), "open-next-")); + const tempDir = await fsp.mkdtemp(path.join(os.tmpdir(), "open-next-")); for (const i of tqdm(Array.from({ length: totalChunks }, (_, i) => i))) { const chunkPath = path.join(tempDir, `cache-chunk-${i}.json`); @@ -467,10 +467,10 @@ async function populateKVIncrementalCache( buildId, cacheType: isFetch ? "fetch" : "cache", }), - value: readFileSync(fullPath, "utf8"), + value: fs.readFileSync(fullPath, "utf8"), })); - writeFileSync(chunkPath, JSON.stringify(kvMapping)); + fs.writeFileSync(chunkPath, JSON.stringify(kvMapping)); runWrangler( buildOpts, @@ -488,7 +488,7 @@ async function populateKVIncrementalCache( } ); - rmSync(chunkPath, { force: true }); + fs.rmSync(chunkPath, { force: true }); } logger.info(`Successfully populated cache with ${assets.length} assets`); @@ -530,7 +530,7 @@ function populateD1TagCache( function populateStaticAssetsIncrementalCache(options: BuildOptions) { logger.info("\nPopulating Workers static assets..."); - cpSync( + fs.cpSync( path.join(options.outputDir, "cache"), path.join(options.outputDir, "assets", STATIC_ASSETS_CACHE_DIR), { recursive: true } From d07c4ecc5af5b85fff90d3c772dff8abc2646aa4 Mon Sep 17 00:00:00 2001 From: Mateusz Grab Date: Fri, 6 Feb 2026 15:14:57 +0100 Subject: [PATCH 09/10] Add warning about potential lack of support on all the platforms --- .changeset/recovered-rclone-batch-upload.md | 4 ++++ packages/cloudflare/src/api/cloudflare-context.ts | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.changeset/recovered-rclone-batch-upload.md b/.changeset/recovered-rclone-batch-upload.md index afa709dcc..4a5ebc7d3 100644 --- a/.changeset/recovered-rclone-batch-upload.md +++ b/.changeset/recovered-rclone-batch-upload.md @@ -39,3 +39,7 @@ You can also set the environment variables for CI builds. **Note:** You can follow documentation https://developers.cloudflare.com/r2/api/tokens/ for creating API tokens with appropriate permissions for R2 access. + +**WARNING:** + +Approach with rclone batch upload might not be supported on all the platforms. diff --git a/packages/cloudflare/src/api/cloudflare-context.ts b/packages/cloudflare/src/api/cloudflare-context.ts index 8400ed704..66944522f 100644 --- a/packages/cloudflare/src/api/cloudflare-context.ts +++ b/packages/cloudflare/src/api/cloudflare-context.ts @@ -87,7 +87,8 @@ declare global { // Cloudflare account id - needed for skew protection and R2 batch population CF_ACCOUNT_ID?: string; - // R2 API credentials for batch cache population (optional, enables faster uploads) + // R2 API credentials + // WARNING: these credentials are used only for rclone R2 batch upload, which might not be supported on all the platforms R2_ACCESS_KEY_ID?: string; R2_SECRET_ACCESS_KEY?: string; } From def3d3beed4f4281afa9bb45a70f6c71e55b19e7 Mon Sep 17 00:00:00 2001 From: Victor Berchet Date: Sun, 8 Feb 2026 06:55:48 +0100 Subject: [PATCH 10/10] fixup! updates --- .changeset/recovered-rclone-batch-upload.md | 31 +-- .../cloudflare/src/api/cloudflare-context.ts | 6 +- .../src/cli/commands/populate-cache.spec.ts | 208 ++++++------------ .../src/cli/commands/populate-cache.ts | 201 ++++++++--------- 4 files changed, 169 insertions(+), 277 deletions(-) diff --git a/.changeset/recovered-rclone-batch-upload.md b/.changeset/recovered-rclone-batch-upload.md index 4a5ebc7d3..cc523fa3e 100644 --- a/.changeset/recovered-rclone-batch-upload.md +++ b/.changeset/recovered-rclone-batch-upload.md @@ -2,31 +2,27 @@ "@opennextjs/cloudflare": minor --- -feature: optional batch upload via rclone for fast R2 cache population. - -This PR implement batch upload support for R2 cache population via rclone, which is not subject to the R2 REST API limits. +feature: optional batch upload via `rclone` for fast R2 cache population. **Key Changes:** -1. **Optional Batch Upload**: Configure R2 credentials via `.env` or secrets to opt in to rclone based batch uploads: +1. **Optional `rclone` Upload**: Configure R2 credentials to opt in to `rclone` based batch uploads: - - `R2_ACCESS_KEY_ID` - - `R2_SECRET_ACCESS_KEY` - - `CF_ACCOUNT_ID` + - `R2_ACCESS_KEY_ID` + - `R2_SECRET_ACCESS_KEY` + - `CF_ACCOUNT_ID` -2. **Automatic Detection**: When credentials are detected, batch upload is automatically used +2. **Automatic Detection**: When credentials are detected and the target is `remote`, `rclone` batch upload is automatically used. Local targets always use `wrangler r2 bulk put` directly. -3. **Smart Fallback**: If credentials are not configured, the CLI falls back to standard Wrangler uploads with a helpful message about enabling batch upload to bypass Account Level Rate Limits +3. **Smart Fallback**: If credentials are not configured or if `rclone` fails, the CLI falls back to `wrangler r2 bulk put`. **Benefits (when batch upload is enabled):** -- Parallel transfer capabilities (32 concurrent transfers) -- Reduced API calls to Cloudflare -- Bypassing Account Level Rate Limits +`rclone` uses the S3 API which is not subject to the REST API limits **Usage:** -Add the credentials in a `.env`/`.dev.vars` file in your project root: +Add the secrets in a `.env`/`.dev.vars` file in your project root, ```bash R2_ACCESS_KEY_ID=your_key @@ -36,10 +32,7 @@ CF_ACCOUNT_ID=your_account You can also set the environment variables for CI builds. -**Note:** - -You can follow documentation https://developers.cloudflare.com/r2/api/tokens/ for creating API tokens with appropriate permissions for R2 access. - -**WARNING:** +**Notes:** -Approach with rclone batch upload might not be supported on all the platforms. +- You can follow documentation for creating API tokens with appropriate permissions for R2 access. +- `rclone` may not be supported on all platforms. diff --git a/packages/cloudflare/src/api/cloudflare-context.ts b/packages/cloudflare/src/api/cloudflare-context.ts index 66944522f..9c76abfde 100644 --- a/packages/cloudflare/src/api/cloudflare-context.ts +++ b/packages/cloudflare/src/api/cloudflare-context.ts @@ -87,8 +87,10 @@ declare global { // Cloudflare account id - needed for skew protection and R2 batch population CF_ACCOUNT_ID?: string; - // R2 API credentials - // WARNING: these credentials are used only for rclone R2 batch upload, which might not be supported on all the platforms + // R2 S3 API credentials used by `rclone` + // NOTES: + // - `rclone` may not be supported on all platforms + // - `rclone` is not supported in development R2_ACCESS_KEY_ID?: string; R2_SECRET_ACCESS_KEY?: string; } diff --git a/packages/cloudflare/src/cli/commands/populate-cache.spec.ts b/packages/cloudflare/src/cli/commands/populate-cache.spec.ts index 6743d5bc5..abe0c6cc6 100644 --- a/packages/cloudflare/src/cli/commands/populate-cache.spec.ts +++ b/packages/cloudflare/src/cli/commands/populate-cache.spec.ts @@ -2,9 +2,14 @@ import { mkdirSync, writeFileSync } from "node:fs"; import path from "node:path"; import type { BuildOptions } from "@opennextjs/aws/build/helper.js"; +import type { OpenNextConfig } from "@opennextjs/aws/types/open-next.js"; import mockFs from "mock-fs"; +import rclone from "rclone.js"; import { afterAll, afterEach, beforeAll, describe, expect, test, vi } from "vitest"; +import type { Unstable_Config as WranglerConfig } from "wrangler"; +import { runWrangler } from "../utils/run-wrangler.js"; +import type { WorkerEnvVar } from "./helpers.js"; import { getCacheAssets, populateCache } from "./populate-cache.js"; describe("getCacheAssets", () => { @@ -78,7 +83,7 @@ vi.mock("./helpers.js", () => ({ quoteShellMeta: vi.fn((s) => s), })); -// Mock rclone.js promises API to simulate successful copy operations by default +// Mock `rclone.js` promises API to simulate successful copy operations by default vi.mock("rclone.js", () => ({ default: { promises: { @@ -88,196 +93,109 @@ vi.mock("rclone.js", () => ({ })); describe("populateCache", () => { - // Test fixtures - const createTestBuildOptions = (): BuildOptions => - ({ - outputDir: "/test/output", - }) as BuildOptions; - - const createTestOpenNextConfig = () => ({ - default: { - override: { - incrementalCache: "cf-r2-incremental-cache", - }, - }, - }); + describe("R2 incremental cache", () => { + const buildOptions = { outputDir: "/test/output" } as BuildOptions; - const createTestWranglerConfig = () => ({ - r2_buckets: [ - { - binding: "NEXT_INC_CACHE_R2_BUCKET", - bucket_name: "test-bucket", + const openNextConfig = { + default: { + override: { + incrementalCache: () => Promise.resolve({ name: "cf-r2-incremental-cache" }), + }, }, - ], - }); - - const createTestPopulateCacheOptions = () => ({ - target: "local" as const, - shouldUsePreviewId: false, - }); + } as OpenNextConfig; - const setupMockFileSystem = () => { - mockFs({ - "/test/output": { - cache: { - buildID: { - path: { - to: { - "test.cache": JSON.stringify({ data: "test" }), + const wranglerConfig = { + r2_buckets: [ + { + binding: "NEXT_INC_CACHE_R2_BUCKET", + bucket_name: "test-bucket", + }, + ], + } as WranglerConfig; + + const r2Credentials = { + R2_ACCESS_KEY_ID: "test_access_key", + R2_SECRET_ACCESS_KEY: "test_secret_key", + CF_ACCOUNT_ID: "test_account_id", + } as WorkerEnvVar; + + const setupMockFileSystem = () => { + mockFs({ + "/test/output": { + cache: { + buildID: { + path: { + to: { + "test.cache": JSON.stringify({ data: "test" }), + }, }, }, }, }, - }, - }); - }; + }); + }; - describe("R2 incremental cache", () => { afterEach(() => { mockFs.restore(); vi.unstubAllEnvs(); }); - test("uses sequential upload for local target (skips batch upload)", async () => { - const { runWrangler } = await import("../utils/run-wrangler.js"); - const rcloneModule = (await import("rclone.js")).default; - + test("uses `wrangler r2 bulk put` for local target", async () => { setupMockFileSystem(); vi.mocked(runWrangler).mockClear(); - vi.mocked(rcloneModule.promises.copy).mockClear(); + vi.mocked(rclone.promises.copy).mockClear(); - // Test with local target - should skip batch upload even with credentials await populateCache( - createTestBuildOptions(), - createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any - createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any + buildOptions, + openNextConfig, + wranglerConfig, { target: "local" as const, shouldUsePreviewId: false }, - { - R2_ACCESS_KEY_ID: "test_access_key", - R2_SECRET_ACCESS_KEY: "test_secret_key", - CF_ACCOUNT_ID: "test_account_id", - } as any // eslint-disable-line @typescript-eslint/no-explicit-any - ); - - // Should use sequential upload (runWrangler), not batch upload (rclone.js) - expect(runWrangler).toHaveBeenCalled(); - expect(rcloneModule.promises.copy).not.toHaveBeenCalled(); - }); - - test("uses sequential upload when R2 credentials are not provided", async () => { - const { runWrangler } = await import("../utils/run-wrangler.js"); - const rcloneModule = (await import("rclone.js")).default; - - setupMockFileSystem(); - vi.mocked(runWrangler).mockClear(); - vi.mocked(rcloneModule.promises.copy).mockClear(); - - // Test uses partial types for simplicity - full config not needed - // Pass empty envVars to simulate no R2 credentials - await populateCache( - createTestBuildOptions(), - createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any - createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any - createTestPopulateCacheOptions(), - {} as any // eslint-disable-line @typescript-eslint/no-explicit-any + r2Credentials ); expect(runWrangler).toHaveBeenCalled(); - expect(rcloneModule.promises.copy).not.toHaveBeenCalled(); + expect(rclone.promises.copy).not.toHaveBeenCalled(); }); - test("uses batch upload with temporary config for remote target when R2 credentials are provided", async () => { - const rcloneModule = (await import("rclone.js")).default; - + test("uses `rclone` for remote target when R2 credentials are provided", async () => { setupMockFileSystem(); - vi.mocked(rcloneModule.promises.copy).mockClear(); + vi.mocked(rclone.promises.copy).mockClear(); - // Test uses partial types for simplicity - full config not needed - // Pass envVars with R2 credentials and remote target to enable batch upload await populateCache( - createTestBuildOptions(), - createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any - createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any + buildOptions, + openNextConfig, + wranglerConfig, { target: "remote" as const, shouldUsePreviewId: false }, - { - R2_ACCESS_KEY_ID: "test_access_key", - R2_SECRET_ACCESS_KEY: "test_secret_key", - CF_ACCOUNT_ID: "test_account_id", - } as any // eslint-disable-line @typescript-eslint/no-explicit-any + r2Credentials ); - // Verify batch upload was used with correct parameters and temporary config - expect(rcloneModule.promises.copy).toHaveBeenCalledWith( + expect(rclone.promises.copy).toHaveBeenCalledWith( expect.any(String), // staging directory "r2:test-bucket", expect.objectContaining({ progress: true, - transfers: 16, - checkers: 8, + transfers: expect.any(Number), + checkers: expect.any(Number), env: expect.objectContaining({ - RCLONE_CONFIG: expect.stringMatching(/rclone-config-\d+\.conf$/), + RCLONE_CONFIG: expect.any(String), // `rclone` config content with R2 credentials }), }) ); }); - test("handles rclone errors with status > 0 for remote target", async () => { - const { runWrangler } = await import("../utils/run-wrangler.js"); - const rcloneModule = (await import("rclone.js")).default; - - setupMockFileSystem(); - - // Mock rclone failure - Promise rejection - vi.mocked(rcloneModule.promises.copy).mockRejectedValueOnce( - new Error("rclone copy failed with exit code 7") - ); - - vi.mocked(runWrangler).mockClear(); - - // Pass envVars with R2 credentials and remote target to enable batch upload (which will fail) - await populateCache( - createTestBuildOptions(), - createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any - createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any - { target: "remote" as const, shouldUsePreviewId: false }, - { - R2_ACCESS_KEY_ID: "test_access_key", - R2_SECRET_ACCESS_KEY: "test_secret_key", - CF_ACCOUNT_ID: "test_account_id", - } as any // eslint-disable-line @typescript-eslint/no-explicit-any - ); - - // Should fall back to sequential upload when batch upload fails - expect(runWrangler).toHaveBeenCalled(); - }); - - test("handles rclone errors with stderr output for remote target", async () => { - const { runWrangler } = await import("../utils/run-wrangler.js"); - const rcloneModule = (await import("rclone.js")).default; - + test("fallback to `wrangler r2 bulk put` when `rclone` fails", async () => { setupMockFileSystem(); - - // Mock rclone error - Promise rejection with stderr message - vi.mocked(rcloneModule.promises.copy).mockRejectedValueOnce( - new Error("ERROR : Failed to copy: AccessDenied: Access Denied (403)") - ); - + vi.mocked(rclone.promises.copy).mockRejectedValueOnce(new Error("rclone copy failed with exit code 7")); vi.mocked(runWrangler).mockClear(); - // Pass envVars with R2 credentials and remote target to enable batch upload (which will fail) await populateCache( - createTestBuildOptions(), - createTestOpenNextConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any - createTestWranglerConfig() as any, // eslint-disable-line @typescript-eslint/no-explicit-any + buildOptions, + openNextConfig, + wranglerConfig, { target: "remote" as const, shouldUsePreviewId: false }, - { - R2_ACCESS_KEY_ID: "test_access_key", - R2_SECRET_ACCESS_KEY: "test_secret_key", - CF_ACCOUNT_ID: "test_account_id", - } as any // eslint-disable-line @typescript-eslint/no-explicit-any + r2Credentials ); - // Should fall back to standard upload when batch upload fails expect(runWrangler).toHaveBeenCalled(); }); }); diff --git a/packages/cloudflare/src/cli/commands/populate-cache.ts b/packages/cloudflare/src/cli/commands/populate-cache.ts index a126f78a7..8e7a86142 100644 --- a/packages/cloudflare/src/cli/commands/populate-cache.ts +++ b/packages/cloudflare/src/cli/commands/populate-cache.ts @@ -107,7 +107,7 @@ export async function populateCache( await populateKVIncrementalCache(buildOpts, wranglerConfig, populateCacheOptions, envVars); break; case STATIC_ASSETS_CACHE_NAME: - populateStaticAssetsIncrementalCache(buildOpts); + await populateStaticAssetsIncrementalCache(buildOpts); break; default: logger.info("Incremental cache does not need populating"); @@ -209,12 +209,72 @@ type PopulateCacheOptions = { }; /** - * Create a temporary rclone configuration file. - * @returns Path to the temporary config file or null if env vars not available + * Populates R2 incremental cache + * + * For remote targets, this function will attempt to use `rclone` first. + * If `rclone` fails, it will fall back to using `wrangler r2 bulk put` command. + * + * For local targets, `wrangler r2 bulk put` command is used directly. + * + * @param buildOpts The normalized build options for the current Open Next build + * @param config The Wrangler configuration object + * @param populateCacheOptions The options for populating the cache + * @param envVars Environment variables to use when populating the cache + * @returns A promise that resolves when the cache population is complete */ -function createTempRcloneConfig(accessKey: string, secretKey: string, accountId: string): string | null { - const tempDir = os.tmpdir(); - const tempConfigPath = path.join(tempDir, `rclone-config-${Date.now()}.conf`); +async function populateR2IncrementalCache( + buildOpts: BuildOptions, + config: WranglerConfig, + populateCacheOptions: PopulateCacheOptions, + envVars: WorkerEnvVar +): Promise { + logger.info("Populating R2 incremental cache..."); + + const binding = config.r2_buckets.find(({ binding }) => binding === R2_CACHE_BINDING_NAME); + if (!binding) { + throw new Error(`No R2 binding "${R2_CACHE_BINDING_NAME}" found!`); + } + + const bucket = binding.bucket_name; + if (!bucket) { + throw new Error(`R2 binding "${R2_CACHE_BINDING_NAME}" should have a 'bucket_name'`); + } + + const prefix = envVars[R2_CACHE_PREFIX_ENV_NAME]; + + const assets = getCacheAssets(buildOpts); + + if (populateCacheOptions.target === "remote") { + // Attempt `rclone` first for remote target + try { + return await populateR2IncrementalCacheWithRclone(bucket, prefix, assets, envVars); + } catch (error) { + logger.warn(`Batch upload failed: ${error instanceof Error ? error.message : error}`); + logger.info("Falling back to wrangler r2 bulk put..."); + } + } + + return await populateR2WithWranglerBulkPut( + buildOpts, + bucket, + prefix, + assets, + populateCacheOptions, + binding.jurisdiction + ); +} + +/** + * Create a temporary `rclone` configuration file. + * @returns Path to the temporary config file + */ +async function createTempRcloneConfig( + accessKey: string, + secretKey: string, + accountId: string +): Promise { + const tempDir = await fsp.mkdtemp(path.join(os.tmpdir(), "rclone-config-")); + const tempConfigPath = path.join(tempDir, "rclone.conf"); const configContent = `[r2] type = s3 @@ -226,56 +286,44 @@ acl = private `; // 0o600 means readable and writable by the file owner - fs.writeFileSync(tempConfigPath, configContent, { mode: 0o600 }); + await fsp.writeFile(tempConfigPath, configContent, { mode: 0o600 }); return tempConfigPath; } /** - * Populate R2 incremental cache using rclone for batch upload - * Uses rclone with parallel transfers to significantly speed up cache population + * Populate R2 incremental cache using `rclone` for batch upload + * Uses `rclone` with parallel transfers to significantly speed up cache population */ -async function populateR2IncrementalCacheWithBatchUpload( +async function populateR2IncrementalCacheWithRclone( bucket: string, prefix: string | undefined, assets: CacheAsset[], envVars: WorkerEnvVar ) { - const accessKey = envVars.R2_ACCESS_KEY_ID || null; - const secretKey = envVars.R2_SECRET_ACCESS_KEY || null; - const accountId = envVars.CF_ACCOUNT_ID || null; + const accessKey = envVars.R2_ACCESS_KEY_ID; + const secretKey = envVars.R2_SECRET_ACCESS_KEY; + const accountId = envVars.CF_ACCOUNT_ID; // Ensure all required env vars are set correctly if (!accessKey || !secretKey || !accountId) { throw new Error( - "Please set R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY, and CF_ACCOUNT_ID environment variables to enable batch upload for remote R2 that bypasses Account Rate Limits." + "All of R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY, and CF_ACCOUNT_ID must be provided to use `rclone`." ); } - logger.info("\nPopulating remote R2 incremental cache using rclone..."); + logger.info("\nPopulating remote R2 incremental cache using `rclone`..."); - // Create temporary config from env vars - required for batch upload - const tempConfigPath = createTempRcloneConfig(accessKey, secretKey, accountId); - if (!tempConfigPath) { - throw new Error("Failed to create temporary rclone config for R2 batch upload."); - } + const tempConfigPath = await createTempRcloneConfig(accessKey, secretKey, accountId); const env = { ...process.env, RCLONE_CONFIG: tempConfigPath, }; - logger.info("Using rclone with R2 credentials from environment variables"); - // Create a staging dir in temp directory with proper key paths - const tempDir = os.tmpdir(); - const stagingDir = path.join(tempDir, `.r2-staging-${Date.now()}`); - - // Track success to ensure cleanup happens correctly - let success = null; + const stagingDir = await fsp.mkdtemp(path.join(os.tmpdir(), "r2-staging-")); try { - fs.mkdirSync(stagingDir, { recursive: true }); - for (const { fullPath, key, buildId, isFetch } of assets) { const cacheKey = computeCacheKey(key, { prefix, @@ -283,47 +331,35 @@ async function populateR2IncrementalCacheWithBatchUpload( cacheType: isFetch ? "fetch" : "cache", }); const destPath = path.join(stagingDir, cacheKey); - fs.mkdirSync(path.dirname(destPath), { recursive: true }); - fs.copyFileSync(fullPath, destPath); + await fsp.mkdir(path.dirname(destPath), { recursive: true }); + await fsp.copyFile(fullPath, destPath); } - // Use rclone.js to sync the R2 - const remote = `r2:${bucket}`; - - // Using rclone.js Promise-based API for the copy operation - await rclone.promises.copy(stagingDir, remote, { + await rclone.promises.copy(stagingDir, `r2:${bucket}`, { progress: true, transfers: 16, checkers: 8, env, }); - logger.info(`Successfully uploaded ${assets.length} assets to R2 using rclone`); - success = true; + logger.info(`Successfully uploaded ${assets.length} assets`); } finally { try { - // Cleanup temporary staging directory - fs.rmSync(stagingDir, { recursive: true, force: true }); + await fsp.rm(stagingDir, { recursive: true, force: true }); } catch { logger.info(`Failed to remove temporary staging directory at ${stagingDir}`); } try { - // Cleanup temporary config file - fs.rmSync(tempConfigPath); + await fsp.rm(path.dirname(tempConfigPath), { recursive: true, force: true }); } catch { logger.warn(`Failed to remove temporary config at ${tempConfigPath}`); } } - - if (!success) { - throw new Error("R2 rclone failed, falling back to wrangler r2 bulk put uploads..."); - } } /** * Populate R2 incremental cache using Wrangler's r2 bulk put command - * Falls back to this method when rclone is not available or fails */ async function populateR2WithWranglerBulkPut( buildOpts: BuildOptions, @@ -344,10 +380,9 @@ async function populateR2WithWranglerBulkPut( file: fullPath, })); - const tempDir = path.join(os.tmpdir(), `open-next-${Date.now()}`); - fs.mkdirSync(tempDir, { recursive: true }); + const tempDir = await fsp.mkdtemp(path.join(os.tmpdir(), "open-next-")); const listFile = path.join(tempDir, `r2-bulk-list.json`); - fs.writeFileSync(listFile, JSON.stringify(objectList)); + await fsp.writeFile(listFile, JSON.stringify(objectList)); const concurrency = Math.max(1, populateCacheOptions.cacheChunkSize ?? 50); const jurisdictionFlag = jurisdiction ? `--jurisdiction ${jurisdiction}` : ""; @@ -371,63 +406,7 @@ async function populateR2WithWranglerBulkPut( } ); - fs.rmSync(listFile, { force: true }); - - logger.info(`Successfully populated cache with ${assets.length} assets`); -} - -async function populateR2IncrementalCache( - buildOpts: BuildOptions, - config: WranglerConfig, - populateCacheOptions: PopulateCacheOptions, - envVars: WorkerEnvVar -) { - logger.info("\nPopulating R2 incremental cache..."); - - const binding = config.r2_buckets.find(({ binding }) => binding === R2_CACHE_BINDING_NAME); - if (!binding) { - throw new Error(`No R2 binding ${JSON.stringify(R2_CACHE_BINDING_NAME)} found!`); - } - - const bucket = binding.bucket_name; - if (!bucket) { - throw new Error(`R2 binding ${JSON.stringify(R2_CACHE_BINDING_NAME)} should have a 'bucket_name'`); - } - - const prefix = envVars[R2_CACHE_PREFIX_ENV_NAME]; - - const assets = getCacheAssets(buildOpts); - - // Force wrangler r2 bulk put for local target - if (populateCacheOptions.target === "local") { - logger.info("Using wrangler r2 bulk put for local R2 (rclone batch upload only works with remote R2)"); - return await populateR2WithWranglerBulkPut( - buildOpts, - bucket, - prefix, - assets, - populateCacheOptions, - binding.jurisdiction - ); - } - - try { - // Attempt batch upload first (using rclone) - only for remote target - return await populateR2IncrementalCacheWithBatchUpload(bucket, prefix, assets, envVars); - } catch (error) { - logger.warn(`Batch upload failed: ${error instanceof Error ? error.message : error}`); - logger.info("Falling back to wrangler r2 bulk put..."); - - // Wrangler r2 bulk put fallback - return await populateR2WithWranglerBulkPut( - buildOpts, - bucket, - prefix, - assets, - populateCacheOptions, - binding.jurisdiction - ); - } + await fsp.rm(listFile, { force: true }); } async function populateKVIncrementalCache( @@ -470,7 +449,7 @@ async function populateKVIncrementalCache( value: fs.readFileSync(fullPath, "utf8"), })); - fs.writeFileSync(chunkPath, JSON.stringify(kvMapping)); + await fsp.writeFile(chunkPath, JSON.stringify(kvMapping)); runWrangler( buildOpts, @@ -488,7 +467,7 @@ async function populateKVIncrementalCache( } ); - fs.rmSync(chunkPath, { force: true }); + await fsp.rm(chunkPath, { force: true }); } logger.info(`Successfully populated cache with ${assets.length} assets`); @@ -527,10 +506,10 @@ function populateD1TagCache( logger.info("\nSuccessfully created D1 table"); } -function populateStaticAssetsIncrementalCache(options: BuildOptions) { +async function populateStaticAssetsIncrementalCache(options: BuildOptions) { logger.info("\nPopulating Workers static assets..."); - fs.cpSync( + await fsp.cp( path.join(options.outputDir, "cache"), path.join(options.outputDir, "assets", STATIC_ASSETS_CACHE_DIR), { recursive: true }