Switched to the new upload API for Collections

etc/vortex.api.md

@@ -681,8 +681,8 @@ export class ZoomableImage extends React$2.Component<IZoomableImageProps, {
 
 // Warnings were encountered during analysis:
 //
-// lib/api.d.ts:8796:3 - (ae-forgotten-export) The symbol "MainPageBody" needs to be exported by the entry point api.d.ts
-// lib/api.d.ts:8797:3 - (ae-forgotten-export) The symbol "MainPageHeader" needs to be exported by the entry point api.d.ts
+// lib/api.d.ts:8798:3 - (ae-forgotten-export) The symbol "MainPageBody" needs to be exported by the entry point api.d.ts
+// lib/api.d.ts:8799:3 - (ae-forgotten-export) The symbol "MainPageHeader" needs to be exported by the entry point api.d.ts
 
 // (No @packageDocumentation comment for this package)
 

packages/nexus-api-v3/src/client.ts

@@ -47,6 +47,11 @@ export function createNexusV3Client(options: NexusV3ClientOptions) {
   return {
     ...client,
 
+    /**
+     * Creates a single-part upload session. The returned `presigned_url` is
+     * signed over headers it does not report — see `uploadHeadersFor` for the
+     * values the subsequent PUT has to carry.
+     */
     async createUpload(sizeBytes: number, filename: string) {
       const { data, error, response } = await client.POST("/uploads", {
         body: { size_bytes: sizeBytes, filename },
@@ -55,6 +60,11 @@ export function createNexusV3Client(options: NexusV3ClientOptions) {
       return data.data;
     },
 
+    /**
+     * Creates a multipart upload session, following the Amazon S3 multipart
+     * specification. Its presigned URLs carry the same unreported signed
+     * headers as `createUpload` — see `uploadHeadersFor`.
+     */
     async createMultipartUpload(sizeBytes: number, filename: string) {
       const { data, error, response } = await client.POST("/uploads/multipart", {
         body: { size_bytes: sizeBytes, filename },

packages/nexus-api-v3/src/index.ts

@@ -1,4 +1,5 @@
 export type { paths, components, operations } from "./generated/nexus-api-v3";
 export { createNexusV3Client, type NexusV3Client, type NexusV3ClientOptions } from "./client";
 export { V3ApiError } from "./errors";
+export { uploadHeadersFor, type UploadHeaders } from "./uploadHeaders";
 export type { Middleware as NexusV3Middleware } from "openapi-fetch";

packages/nexus-api-v3/src/uploadHeaders.test.ts

@@ -0,0 +1,18 @@
+import { describe, it, expect } from "vitest";
+
+import { uploadHeadersFor } from "./uploadHeaders";
+
+describe("uploadHeadersFor", () => {
+  it("derives a content-disposition from the session filename", () => {
+    expect(uploadHeadersFor("collection_1.7z")).toEqual({
+      contentType: "application/octet-stream",
+      contentDisposition: 'attachment; filename="collection_1.7z"',
+    });
+  });
+
+  it("quotes the filename so spaces survive", () => {
+    expect(uploadHeadersFor("my collection.7z").contentDisposition).toBe(
+      'attachment; filename="my collection.7z"',
+    );
+  });
+});

packages/nexus-api-v3/src/uploadHeaders.ts

@@ -0,0 +1,17 @@
+/**
+ * Headers a v3 upload session's presigned URL covers with its signature.
+ */
+export type UploadHeaders = {
+  contentType: string;
+  contentDisposition: string;
+};
+
+/**
+ * The header values an upload session was presigned with.
+ */
+export function uploadHeadersFor(filename: string): UploadHeaders {
+  return {
+    contentType: "application/octet-stream",
+    contentDisposition: `attachment; filename="${filename}"`,
+  };
+}

src/main/src/downloading/downloader.test.integration.ts

@@ -10,10 +10,10 @@ import { RateLimiter } from "limiter";
 import { CookieJar } from "tough-cookie";
 import { describe, it, expect, vi, beforeAll, afterAll, test } from "vitest";
 
+import { defaultRetryStrategy } from "../transfer/retry";
 import { download, type TimeoutOptions } from "./downloader";
 import { ProgressReporter } from "./progress";
 import { urlResolver } from "./resolver";
-import { defaultRetryStrategy } from "./retry";
 import {
   type TestServer,
   type RequestHandler,

src/main/src/downloading/downloader.ts

@@ -1,7 +1,7 @@
 import { type FileHandle as NodeFileHandle, access, open } from "node:fs/promises";
 import type { IncomingHttpHeaders } from "node:http";
 
-import { getErrorCode, unknownToError } from "@vortex/shared";
+import { getErrorCode } from "@vortex/shared";
 import type {
   ByteRange,
   Chunk,
@@ -12,17 +12,20 @@ import type {
   RetryStrategy,
 } from "@vortex/shared/download";
 import { DownloadError } from "@vortex/shared/errors";
-import type { Got, Headers, Delays as GotTimeoutOptions, ExtendOptions } from "got";
+import type { Got, Headers, ExtendOptions } from "got";
 import got from "got";
 import type { RateLimiter } from "limiter";
 import PQueue from "p-queue";
 import type { CookieJar } from "tough-cookie";
 
-import { isCancellation, toNetworkError } from "./errors";
+import { isCancellation } from "../transfer/cancellation";
+import { withRetry } from "../transfer/retry";
+import type { TimeoutOptions } from "../transfer/timeouts";
+import { createGotTimeoutOptions } from "../transfer/timeouts";
+import { toNetworkError } from "./errors";
 import type { ProgressReporter } from "./progress";
 import type { NormalizedResource } from "./resolver";
 import { normalize } from "./resolver";
-import { sleep } from "./retry";
 
 export const defaultChunkConcurrency = 4;
 
@@ -32,17 +35,7 @@ export type Checkpoint = {
   completedRanges: ByteRange[];
 };
 
-export type TimeoutOptions = {
-  // TODO: use Temporal API
-  /** Timeout for DNS lookup (ms). */
-  lookup: number;
-
-  /** Timeout for DNS lookup + TCP connect + TLS handshake (ms). */
-  connect: number;
-
-  /** Timeout between received data packets before treating the connection as stalled (ms). */
-  stall: number;
-};
+export type { TimeoutOptions };
 
 /** @internal */
 export async function download<T>(
@@ -358,18 +351,6 @@ function createGotStream(
   return stream;
 }
 
-function createGotTimeoutOptions(timeout?: TimeoutOptions): GotTimeoutOptions | undefined {
-  if (!timeout) return undefined;
-
-  return {
-    lookup: timeout.lookup,
-    connect: timeout.connect,
-    secureConnect: timeout.connect,
-    socket: timeout.stall,
-    response: timeout.stall,
-  };
-}
-
 async function consumeTokens(
   limiter: RateLimiter,
   bytes: number,
@@ -472,32 +453,6 @@ async function downloadChunk(
   });
 }
 
-/**
- * Retry helper that re-invokes `fn` according to the given strategy.
- * Cancellations are never retried. Uses abort-aware sleep so backoff
- * delays are interrupted when the signal fires.
- */
-async function withRetry<T>(
-  fn: () => Promise<T>,
-  strategy?: RetryStrategy,
-  abortSignal?: AbortSignal,
-): Promise<T> {
-  if (!strategy) return await fn();
-
-  let attempt = 0;
-  while (true) {
-    try {
-      return await fn();
-    } catch (err) {
-      if (isCancellation(err)) throw err;
-      attempt++;
-      const verdict = strategy({ attempt, error: unknownToError(err) });
-      if (!verdict.retry) throw err;
-      await sleep(verdict.delayMs, abortSignal);
-    }
-  }
-}
-
 function createHeaders(
   etag: string | undefined,
   chunk: Chunk | undefined,

src/main/src/downloading/errors.ts

@@ -1,15 +1,6 @@
 import type { ResolvedEndpoint } from "@vortex/shared/download";
 import { DownloadError } from "@vortex/shared/errors";
-import { TimeoutError, HTTPError, RequestError, AbortError } from "got";
-
-export function isCancellation(err: unknown): boolean {
-  // NOTE(erri120): The `got` package throws a custom `AbortError` class on cancellation
-  if (err instanceof AbortError) return true;
-
-  // NOTE(erri120): The `p-queue` package and anything else using `AbortController`
-  // throw a `DOMException` with `name = "AbortError` instead.
-  return err instanceof DOMException && err.name === "AbortError";
-}
+import { TimeoutError, HTTPError, RequestError } from "got";
 
 export function toNetworkError(endpoint: URL | ResolvedEndpoint, err: unknown): DownloadError {
   const url = endpoint instanceof URL ? endpoint : endpoint.url;

src/main/src/downloading/manager.ts

@@ -16,10 +16,10 @@ import PQueue from "p-queue";
 import type { CookieJar } from "tough-cookie";
 
 import { log } from "../logging";
+import { defaultRetryStrategy } from "../transfer/retry";
 import type { TimeoutOptions } from "./downloader";
 import { download } from "./downloader";
 import { ProgressReporter } from "./progress";
-import { defaultRetryStrategy } from "./retry";
 
 export type DownloadHandle<T = unknown> = {
   /** Globally unique identifier for this download. */

src/main/src/main.ts

@@ -60,6 +60,8 @@ import { log } from "./logging";
 import StylesheetCompiler from "./stylesheetCompiler";
 import { initTelemetryIpcHandler } from "./telemetry/ipcHandler";
 import { createMainTelemetryProvider } from "./telemetry/setup";
+import { init as initUploadIpc } from "./uploading/ipc";
+import { UploadManager } from "./uploading/manager";
 
 process.env["UV_THREADPOOL_SIZE"] = (os.cpus().length * 2).toString();
 
@@ -297,9 +299,11 @@ async function main(): Promise<void> {
   process.on("unhandledRejection", handleError);
 
   const downloadManager = new DownloadManager({ concurrency: 1 });
+  const uploadManager = new UploadManager({ userAgent: `Vortex/${app.getVersion()}` });
 
   initIpcHandlers();
   initDownloadIpc(downloadManager);
+  initUploadIpc(uploadManager);
   initAdaptorHost().catch((err: unknown) => {
     log("warn", "Failed to initialize adaptor host", {
       error: err instanceof Error ? err.message : "unknown error",

src/main/src/transfer/cancellation.ts

@@ -0,0 +1,14 @@
+/**
+ * Cancellation detection shared by the download and upload transfer paths.
+ * Both drive `got` with an `AbortSignal`, so both see the same two shapes.
+ */
+import { AbortError } from "got";
+
+export function isCancellation(err: unknown): boolean {
+  // NOTE(erri120): The `got` package throws a custom `AbortError` class on cancellation
+  if (err instanceof AbortError) return true;
+
+  // NOTE(erri120): The `p-queue` package and anything else using `AbortController`
+  // throw a `DOMException` with `name = "AbortError` instead.
+  return err instanceof DOMException && err.name === "AbortError";
+}

src/main/src/downloading/retry.ts → src/main/src/transfer/retry.ts

@@ -1,7 +1,10 @@
+import { unknownToError } from "@vortex/shared";
 import type { RetryContext, RetryStrategy, RetryVerdict } from "@vortex/shared/download";
-import { DownloadError } from "@vortex/shared/errors";
+import { DownloadError, UploadError } from "@vortex/shared/errors";
 import { HTTPError } from "got";
 
+import { isCancellation } from "./cancellation";
+
 const retryableErrorCodes = new Set([
   // Connection timed out (POSIX.1-2001).
   "ETIMEDOUT",
@@ -52,7 +55,7 @@ export function defaultRetryStrategy(
 }
 
 function isRetryableError(err: Error, codes: Set<string>, statusCodes: Set<number>): boolean {
-  if (err instanceof DownloadError) {
+  if (err instanceof DownloadError || err instanceof UploadError) {
     if (
       err.code === "fs-error" ||
       err.code === "protocol-violation" ||
@@ -78,6 +81,32 @@ function isRetryableError(err: Error, codes: Set<string>, statusCodes: Set<numbe
   return false;
 }
 
+/**
+ * Retry helper that re-invokes `fn` according to the given strategy.
+ * Cancellations are never retried. Uses abort-aware sleep so backoff
+ * delays are interrupted when the signal fires.
+ */
+export async function withRetry<T>(
+  fn: () => Promise<T>,
+  strategy?: RetryStrategy,
+  abortSignal?: AbortSignal,
+): Promise<T> {
+  if (!strategy) return await fn();
+
+  let attempt = 0;
+  while (true) {
+    try {
+      return await fn();
+    } catch (err) {
+      if (isCancellation(err)) throw err;
+      attempt++;
+      const verdict = strategy({ attempt, error: unknownToError(err) });
+      if (!verdict.retry) throw err;
+      await sleep(verdict.delayMs, abortSignal);
+    }
+  }
+}
+
 /**
  * Abort-aware sleep. Resolves after `ms` milliseconds, or rejects
  * immediately if the signal is already aborted or becomes aborted

src/main/src/transfer/timeouts.ts

@@ -0,0 +1,25 @@
+import type { Delays as GotTimeoutOptions } from "got";
+
+export type TimeoutOptions = {
+  // TODO: use Temporal API
+  /** Timeout for DNS lookup (ms). */
+  lookup: number;
+
+  /** Timeout for DNS lookup + TCP connect + TLS handshake (ms). */
+  connect: number;
+
+  /** Timeout between received data packets before treating the connection as stalled (ms). */
+  stall: number;
+};
+
+export function createGotTimeoutOptions(timeout?: TimeoutOptions): GotTimeoutOptions | undefined {
+  if (!timeout) return undefined;
+
+  return {
+    lookup: timeout.lookup,
+    connect: timeout.connect,
+    secureConnect: timeout.connect,
+    socket: timeout.stall,
+    response: timeout.stall,
+  };
+}