Merge pull request #3398 from trycompai/feat/pentest-auto-retry-failed-scans

feat(api): auto-retry failed pentest scans and clean up failure messages

Author: tofikwest

apps/api/src/security-penetration-tests/pentest-lineage.util.spec.ts

@@ -0,0 +1,113 @@
+import {
+  collapsedStatus,
+  MAX_ATTEMPTS,
+  RETRY_GRACE_MS,
+} from './pentest-lineage.util';
+
+describe('collapsedStatus', () => {
+  const now = 1_000_000_000_000;
+
+  it('passes through completed', () => {
+    expect(
+      collapsedStatus({
+        activeStatus: 'completed',
+        attemptNumber: 2,
+        failedAtMs: null,
+        nowMs: now,
+      }),
+    ).toBe('completed');
+  });
+
+  it.each(['provisioning', 'cloning', 'running'] as const)(
+    'passes through in-progress status %s',
+    (status) => {
+      expect(
+        collapsedStatus({
+          activeStatus: status,
+          attemptNumber: 1,
+          failedAtMs: null,
+          nowMs: now,
+        }),
+      ).toBe(status);
+    },
+  );
+
+  it('never masks cancelled (staff cancel is terminal, never retried)', () => {
+    expect(
+      collapsedStatus({
+        activeStatus: 'cancelled',
+        attemptNumber: 1,
+        failedAtMs: now - 1000,
+        nowMs: now,
+      }),
+    ).toBe('cancelled');
+  });
+
+  describe('failed', () => {
+    it('masks a fresh failed non-final attempt as provisioning', () => {
+      expect(
+        collapsedStatus({
+          activeStatus: 'failed',
+          attemptNumber: 1,
+          failedAtMs: now - 5_000, // well within grace
+          nowMs: now,
+        }),
+      ).toBe('provisioning');
+    });
+
+    it('masks the second attempt too', () => {
+      expect(
+        collapsedStatus({
+          activeStatus: 'failed',
+          attemptNumber: 2,
+          failedAtMs: now - 5_000,
+          nowMs: now,
+        }),
+      ).toBe('provisioning');
+    });
+
+    it('reveals failed once the lineage is exhausted (final attempt)', () => {
+      expect(
+        collapsedStatus({
+          activeStatus: 'failed',
+          attemptNumber: MAX_ATTEMPTS,
+          failedAtMs: now - 5_000,
+          nowMs: now,
+        }),
+      ).toBe('failed');
+    });
+
+    it('reveals failed after the grace window elapses without a retry', () => {
+      expect(
+        collapsedStatus({
+          activeStatus: 'failed',
+          attemptNumber: 1,
+          failedAtMs: now - (RETRY_GRACE_MS + 1),
+          nowMs: now,
+        }),
+      ).toBe('failed');
+    });
+
+    it('reveals failed exactly at the grace-window boundary', () => {
+      expect(
+        collapsedStatus({
+          activeStatus: 'failed',
+          attemptNumber: 1,
+          failedAtMs: now - RETRY_GRACE_MS,
+          nowMs: now,
+        }),
+      ).toBe('failed');
+    });
+
+    it('reveals failed when the failure time is unknown (cannot bound the mask)', () => {
+      expect(
+        collapsedStatus({
+          activeStatus: 'failed',
+          attemptNumber: 1,
+          failedAtMs: null,
+          nowMs: now,
+        }),
+      ).toBe('failed');
+    });
+  });
+});

apps/api/src/security-penetration-tests/pentest-lineage.util.ts

@@ -0,0 +1,66 @@
+/**
+ * Pure helpers for the pentest retry-lineage feature.
+ *
+ * A scan is a "lineage": the original run plus up to `MAX_ATTEMPTS - 1`
+ * automatic retries, all sharing one root run id. Customers only ever see the
+ * root; our API resolves it to the active (highest-numbered) attempt and only
+ * reports a terminal failure once the whole lineage is exhausted.
+ */
+
+export type PentestRunStatus =
+  | 'provisioning'
+  | 'cloning'
+  | 'running'
+  | 'completed'
+  | 'failed'
+  | 'cancelled';
+
+/** Original attempt + 2 automatic retries. */
+export const MAX_ATTEMPTS = 3;
+
+/**
+ * How long a failed non-final attempt is masked as in-progress while its
+ * retry is spawned. Comfortably covers provider webhook-delivery latency; if a
+ * retry never materializes within this window the real failure is revealed so
+ * the run can't hang on a fake "running" state forever.
+ */
+export const RETRY_GRACE_MS = 10 * 60 * 1000;
+
+/**
+ * Computes the customer-facing status for a lineage from its active (highest)
+ * attempt. The active attempt is authoritative because we only ever retry
+ * `failed` runs — so no earlier attempt can be `completed`/`running`.
+ *
+ * - `completed` / in-progress statuses pass through unchanged.
+ * - `cancelled` is terminal and never retried → always revealed.
+ * - `failed`:
+ *     - lineage exhausted (`attemptNumber >= maxAttempts`) → revealed.
+ *     - otherwise masked as `provisioning` (a retry is expected) while inside
+ *       the grace window; revealed once the window elapses without a retry
+ *       appearing, or when the failure time is unknown (can't bound the mask).
+ */
+export function collapsedStatus(params: {
+  activeStatus: PentestRunStatus;
+  attemptNumber: number;
+  failedAtMs: number | null;
+  nowMs: number;
+  maxAttempts?: number;
+  graceMs?: number;
+}): PentestRunStatus {
+  const { activeStatus, attemptNumber, failedAtMs, nowMs } = params;
+  const maxAttempts = params.maxAttempts ?? MAX_ATTEMPTS;
+  const graceMs = params.graceMs ?? RETRY_GRACE_MS;
+
+  // Only `failed` triggers an auto-retry, so only `failed` is ever masked.
+  if (activeStatus !== 'failed') {
+    return activeStatus;
+  }
+
+  if (attemptNumber >= maxAttempts) {
+    return 'failed';
+  }
+  if (failedAtMs === null || nowMs - failedAtMs >= graceMs) {
+    return 'failed';
+  }
+  return 'provisioning';
+}

apps/api/src/security-penetration-tests/pentest-run-error.util.spec.ts

@@ -0,0 +1,95 @@
+import { toCustomerFacingError } from './pentest-run-error.util';
+
+describe('toCustomerFacingError', () => {
+  const infraMessagePrefix =
+    'The scan was interrupted by a temporary infrastructure issue';
+
+  describe('infra / transient failures', () => {
+    const infraErrors = [
+      'Sandbox container deleted by Daytona infrastructure (age 35m). Desired-state=started but container gone — likely Daytona backup/restore race condition.',
+      'Run timed out — sandbox became unresponsive',
+      'Timed out with no sandbox after 9m',
+      'Hard timeout: 4m (cap 120m); Daytona unreachable',
+    ];
+
+    it.each(infraErrors)(
+      'maps infra error to the transient message: %s',
+      (raw) => {
+        expect(toCustomerFacingError(raw)).toContain(infraMessagePrefix);
+      },
+    );
+
+    it('never echoes the raw provider text', () => {
+      const raw =
+        'Sandbox container deleted by Daytona infrastructure (age 35m). backup/restore race condition.';
+      const message = toCustomerFacingError(raw);
+      expect(message.toLowerCase()).not.toContain('daytona');
+      expect(message.toLowerCase()).not.toContain('sandbox');
+      expect(message.toLowerCase()).not.toContain('container');
+    });
+  });
+
+  describe('genuine runtime cap', () => {
+    it('humanizes a 720m cap as 12 hours', () => {
+      const message = toCustomerFacingError(
+        'Hard timeout: workflow ran 722m (cap 720m)',
+      );
+      expect(message).toContain('maximum runtime of 12 hours');
+      expect(message).toContain("won't be charged");
+    });
+
+    it('humanizes a 120m cap as 2 hours', () => {
+      const message = toCustomerFacingError(
+        'Hard timeout: workflow ran 130m (cap 120m)',
+      );
+      expect(message).toContain('maximum runtime of 2 hours');
+    });
+
+    it('renders a single-hour cap without pluralizing', () => {
+      const message = toCustomerFacingError(
+        'Hard timeout: workflow ran 61m (cap 60m)',
+      );
+      expect(message).toContain('maximum runtime of 1 hour');
+      expect(message).not.toContain('1 hours');
+    });
+
+    it('falls back to minutes for a non-hour cap', () => {
+      const message = toCustomerFacingError(
+        'Hard timeout: workflow ran 95m (cap 90m)',
+      );
+      expect(message).toContain('maximum runtime of 90 minutes');
+    });
+
+    it('renders a single-minute cap without pluralizing', () => {
+      const message = toCustomerFacingError(
+        'Hard timeout: workflow ran 2m (cap 1m)',
+      );
+      expect(message).toContain('maximum runtime of 1 minute');
+      expect(message).not.toContain('1 minutes');
+    });
+
+    it('classifies the "; Daytona unreachable" variant as infra, not runtime cap', () => {
+      const message = toCustomerFacingError(
+        'Hard timeout: 4m (cap 120m); Daytona unreachable',
+      );
+      expect(message).toContain(infraMessagePrefix);
+      expect(message).not.toContain('maximum runtime');
+    });
+  });
+
+  describe('unknown / empty', () => {
+    const genericPrefix = "The scan couldn't be completed";
+
+    it('returns the generic message for an unrecognized error', () => {
+      expect(
+        toCustomerFacingError('some brand new error we have not seen'),
+      ).toContain(genericPrefix);
+    });
+
+    it('returns the generic message for null / undefined / empty', () => {
+      expect(toCustomerFacingError(null)).toContain(genericPrefix);
+      expect(toCustomerFacingError(undefined)).toContain(genericPrefix);
+      expect(toCustomerFacingError('')).toContain(genericPrefix);
+    });
+  });
+});

apps/api/src/security-penetration-tests/pentest-run-error.util.ts

@@ -0,0 +1,89 @@
+/**
+ * Maps a raw provider run-error string into a clean, customer-facing message.
+ *
+ * Penetration tests are executed by an external provider whose backend runs
+ * each scan inside a disposable sandbox. When a run fails, the provider returns
+ * a freeform `error` string that can name our infrastructure vendor, describe
+ * internal engineering conditions ("backup/restore race condition"), or be
+ * arbitrary text — none of which is useful or safe to surface to a customer of
+ * a security product.
+ *
+ * This helper is only applied once a scan's entire retry lineage is exhausted
+ * (every attempt failed), so every message it returns describes a genuine,
+ * final failure. It says the customer "won't be charged" rather than asserting
+ * the refund is already complete — a failed run is always refunded, but the
+ * refund and this read can race, so the phrasing stays true regardless of
+ * timing (net effect: a failed scan is never charged).
+ *
+ * The raw string is never echoed back; unrecognized errors fall through to a
+ * safe generic message.
+ */
+
+const INFRA_TRANSIENT_MESSAGE =
+  'The scan was interrupted by a temporary infrastructure issue and ' +
+  "couldn't be completed. You won't be charged for this scan — please try " +
+  'running it again, and contact support if this keeps happening.';
+
+const GENERIC_MESSAGE =
+  "The scan couldn't be completed. You won't be charged for this scan — " +
+  'please try running it again, or contact support if this keeps happening.';
+
+/**
+ * True for transient infrastructure failures that a re-run typically clears:
+ * the sandbox was deleted, never provisioned, went unresponsive, or the
+ * provisioning path could not reach the sandbox host. The "Daytona
+ * unreachable" hard-timeout variant belongs here (an infra problem during
+ * provisioning), NOT with the genuine runtime cap below — so it must be
+ * checked first.
+ */
+function isInfraTransient(raw: string): boolean {
+  return (
+    /sandbox container deleted/i.test(raw) ||
+    /backup\/restore race/i.test(raw) ||
+    /sandbox became unresponsive/i.test(raw) ||
+    /timed out with no sandbox/i.test(raw) ||
+    /daytona unreachable/i.test(raw)
+  );
+}
+
+/**
+ * Renders a whole number of minutes as a friendly duration: exact hours become
+ * "12 hours" / "1 hour", anything else stays in minutes. Guards against junk
+ * input by falling back to a neutral phrase.
+ */
+function humanizeMinutes(minutes: number): string {
+  if (!Number.isFinite(minutes) || minutes <= 0) {
+    return 'its time limit';
+  }
+  if (minutes % 60 === 0) {
+    const hours = minutes / 60;
+    return `${hours} hour${hours === 1 ? '' : 's'}`;
+  }
+  return `${minutes} minute${minutes === 1 ? '' : 's'}`;
+}
+
+export function toCustomerFacingError(
+  rawError: string | null | undefined,
+): string {
+  const raw = typeof rawError === 'string' ? rawError : '';
+
+  if (isInfraTransient(raw)) {
+    return INFRA_TRANSIENT_MESSAGE;
+  }
+
+  // Genuine runtime cap: the scan used its full time budget without finishing
+  // (e.g. "Hard timeout: workflow ran 722m (cap 720m)"). The infra check above
+  // has already claimed the "; Daytona unreachable" variant.
+  const capMatch =
+    /hard timeout:\s*workflow ran\s+\d+m\s*\(cap\s+(\d+)m\)/i.exec(raw);
+  if (capMatch) {
+    const cap = humanizeMinutes(Number(capMatch[1]));
+    return (
+      `Your scan reached its maximum runtime of ${cap} before completing. ` +
+      "You won't be charged for this scan. Large targets may need a lighter " +
+      'scan depth — contact support if this keeps happening.'
+    );
+  }
+
+  return GENERIC_MESSAGE;
+}