feat(continue): typed continue-gate errors and batch skips instead of silent drops

docs/reference/cli.md

@@ -1113,6 +1113,15 @@ run folders (`config.json` + `trajectory/llm_trajectory.jsonl`) recursively,
 continues each, and prints a JSON batch summary (exits 1 if any continuation
 failed).
 
+Timeout runs that `continue` cannot resume — an agent with no replay ingress,
+or a run with no LLM recording at all — are reported under `skipped`/`skips`
+rather than aborting the batch or counting as failures, so the rest of the
+sweep still runs and the exit status stays 0. Each skip carries a `reason`
+(`unsupported_agent` or `no_llm_recording`) and `recoverable_in_principle`,
+which is `false` when nothing was recorded and no future ingress could help.
+Naming a single unsupported run explicitly (`bench eval continue <folder>`)
+still fails loudly with exit 1.
+
 ```bash
 bench eval continue-batch path/to/jobs-root --tasks-dir path/to/tasks
 ```

src/benchflow/cli/continue_cmd.py

@@ -212,10 +212,12 @@ def continue_batch_cmd(
         import json
 
         from benchflow.continue_run.batch import (
+            BatchContinueResult,
             continue_batch,
             discover_timeout_run_folders,
             summarize_batch,
         )
+        from benchflow.continue_run.run_folder import ContinueUnsupportedError
 
         _apply_dotenv_to_process_env()
         # Fail fast on a bad ROOT instead of treating a typo'd/nonexistent path as
@@ -236,9 +238,26 @@ def continue_batch_cmd(
                 err=True,
             )
             raise typer.Exit(1)
-        folders = discover_timeout_run_folders(root, limit=limit)
+        # Timeout runs `continue` cannot resume are skipped, not fatal — but they
+        # must still be reported, or a sweep silently leaves part of itself
+        # behind and the operator never learns why.
+        skips: list[BatchContinueResult] = []
+
+        def _record_skip(folder: Path, exc: ContinueUnsupportedError) -> None:
+            skips.append(BatchContinueResult.skip(folder, exc))
+
+        folders = discover_timeout_run_folders(root, limit=limit, on_skip=_record_skip)
+        if skips:
+            typer.secho(
+                f"Skipping {len(skips)} timeout run(s) benchflow continue cannot "
+                'resume; see "skips" below for the reason.',
+                fg=typer.colors.YELLOW,
+            )
         if not folders:
-            typer.secho("No timeout run folders found.", fg=typer.colors.YELLOW)
+            if not skips:
+                typer.secho("No timeout run folders found.", fg=typer.colors.YELLOW)
+                return
+            typer.echo(json.dumps(summarize_batch(skips), indent=2))
             return
 
         typer.echo(
@@ -257,8 +276,10 @@ def continue_batch_cmd(
                 proxy_mode=proxy_mode,
             )
         )
-        summary = summarize_batch(results)
+        summary = summarize_batch([*results, *skips])
         typer.echo(json.dumps(summary, indent=2))
+        # Skips are deliberately not failures: exit status still reflects only
+        # continuations that were attempted and went wrong.
         if summary["failed"]:
             raise typer.Exit(1)
 

src/benchflow/continue_run/__init__.py

@@ -23,13 +23,21 @@
 """
 
 from benchflow.continue_run.run_folder import (
+    CONTINUE_SUPPORTED_AGENTS,
+    ContinueUnsupportedError,
+    MissingRecordingError,
     RunFolder,
     RunFolderError,
+    UnsupportedAgentError,
     load_run_folder,
 )
 
 __all__ = [
+    "CONTINUE_SUPPORTED_AGENTS",
+    "ContinueUnsupportedError",
+    "MissingRecordingError",
     "RunFolder",
     "RunFolderError",
+    "UnsupportedAgentError",
     "load_run_folder",
 ]

src/benchflow/continue_run/batch.py

@@ -9,29 +9,65 @@
 from typing import Any
 
 from benchflow.continue_run.orchestrator import ContinueResult, continue_run
-from benchflow.continue_run.run_folder import RunFolderError, load_run_folder
+from benchflow.continue_run.run_folder import (
+    ContinueUnsupportedError,
+    RunFolderError,
+    is_timeout_run,
+    load_run_folder,
+)
 
 ContinueRunner = Callable[..., Awaitable[ContinueResult]]
+#: Notified once per timeout candidate that `benchflow continue` cannot resume.
+SkipObserver = Callable[[Path, ContinueUnsupportedError], None]
 
 
 @dataclass(frozen=True)
 class BatchContinueResult:
-    """Result for one source folder in a batch continuation."""
+    """Result for one source folder in a batch continuation.
+
+    ``skipped`` separates *out of scope* from *failed*: a run whose agent has no
+    replay ingress, or that was never recorded, did not fail — the batch simply
+    could not act on it. Batch callers must not treat a skip as an error.
+    """
 
     folder: Path
     ok: bool
     continued: ContinueResult | None = None
     error: str | None = None
+    skipped: bool = False
+    reason_code: str | None = None
+    recoverable_in_principle: bool | None = None
+
+    @classmethod
+    def skip(cls, folder: Path, exc: ContinueUnsupportedError) -> BatchContinueResult:
+        """Record an out-of-scope run, carrying the gate's typed reason."""
+        return cls(
+            folder=folder,
+            ok=False,
+            error=str(exc),
+            skipped=True,
+            reason_code=exc.reason_code,
+            recoverable_in_principle=exc.recoverable_in_principle,
+        )
 
 
 def discover_timeout_run_folders(
-    root: str | Path, *, limit: int | None = None
+    root: str | Path,
+    *,
+    limit: int | None = None,
+    on_skip: SkipObserver | None = None,
 ) -> list[Path]:
     """Find OpenHands timeout run folders below ``root``.
 
     Discovery is intentionally artifact-based: a candidate must have a
     ``config.json`` and a usable ``trajectory/llm_trajectory.jsonl``. Non-timeout
     runs are skipped by ``load_run_folder(require_timeout=True)``.
+
+    ``on_skip`` is notified for each folder that *is* a timeout candidate but
+    that ``benchflow continue`` cannot resume (unsupported agent, or no LLM
+    recording). Without it those runs vanish silently, and the operator never
+    learns that part of the sweep was left behind. Finished runs from other
+    agents are not reported — they were never candidates.
     """
     root_path = Path(root).expanduser()
     candidates = [root_path] if (root_path / "config.json").is_file() else []
@@ -46,6 +82,10 @@ def discover_timeout_run_folders(
         seen.add(resolved)
         try:
             load_run_folder(folder, require_timeout=True)
+        except ContinueUnsupportedError as exc:
+            if on_skip is not None and is_timeout_run(folder):
+                on_skip(folder, exc)
+            continue
         except RunFolderError:
             continue
         folders.append(folder)
@@ -85,6 +125,10 @@ async def _one(folder: Path) -> BatchContinueResult:
                     strict_divergence=strict_divergence,
                     proxy_mode=proxy_mode,
                 )
+            except ContinueUnsupportedError as exc:
+                # Out of scope, not broken: one unsupported run in a 200-run
+                # sweep must not cost the other 199 their continuation.
+                return BatchContinueResult.skip(folder, exc)
             except Exception as exc:
                 return BatchContinueResult(folder=folder, ok=False, error=str(exc))
             if result.error:
@@ -100,18 +144,35 @@ async def _one(folder: Path) -> BatchContinueResult:
 
 
 def summarize_batch(results: list[BatchContinueResult]) -> dict[str, Any]:
-    """Small JSON-serializable summary for CLI output and dashboards."""
+    """Small JSON-serializable summary for CLI output and dashboards.
+
+    ``skipped``/``skips`` are reported apart from ``failed``/``errors`` so a
+    sweep containing runs `continue` cannot resume is not scored as a batch of
+    failures — and so the operator still sees which runs were left behind, why,
+    and whether a future replay ingress could reach them.
+    """
     ok = [result for result in results if result.ok]
-    failed = [result for result in results if not result.ok]
+    skipped = [result for result in results if not result.ok and result.skipped]
+    failed = [result for result in results if not result.ok and not result.skipped]
     return {
         "total": len(results),
         "succeeded": len(ok),
         "failed": len(failed),
+        "skipped": len(skipped),
         "outputs": [
             str(result.continued.rollout_dir)
             for result in ok
             if result.continued is not None
         ],
+        "skips": [
+            {
+                "folder": str(result.folder),
+                "reason": result.reason_code,
+                "recoverable_in_principle": result.recoverable_in_principle,
+                "detail": result.error,
+            }
+            for result in skipped
+        ],
         "errors": [
             {
                 "folder": str(result.folder),

src/benchflow/continue_run/run_folder.py

@@ -32,11 +32,105 @@
 # continuing rather than a clean pass/fail.
 _TIMEOUT_CATEGORIES = frozenset({"timeout", "idle_timeout"})
 
+# ── which agents ``benchflow continue`` can resume ────────────────────────────
+#
+# Single source of truth: a future replay ingress adds its agent here and
+# nowhere else. The membership is *protocol*-derived, not a policy about open
+# vs closed model weights — see ``_WHY_PROTOCOL`` below for the mechanism.
+CONTINUE_SUPPORTED_AGENTS: frozenset[str] = frozenset({"openhands"})
+
+_TRAJECTORY_RELPATH = "trajectory/llm_trajectory.jsonl"
+
+_WHY_PROTOCOL = (
+    "That set is derived from the replay wire protocol, not from a policy about "
+    "which models are open: the replay proxy serves POST /v1/chat/completions "
+    "only, and the continuation hands the sandbox LLM_BASE_URL / LLM_API_KEY / "
+    "LLM_MODEL, which only the OpenHands agent template consumes. An agent that "
+    "speaks a different wire (Anthropic Messages, OpenAI Responses, Google "
+    "native) would 404 at the proxy and fall back to host credentials, burning a "
+    "full live run that the artifacts would then mislabel as a replay."
+)
+
+_NO_RECORDING_VERDICT = (
+    f"This run has no {_TRAJECTORY_RELPATH}, which is what a subscription-auth "
+    "run looks like: it bypasses the recording gateway, so nothing was captured "
+    "and it can never be continued — no replay ingress, present or future, can "
+    "reconstruct it."
+)
+
+# Offered as a possibility, not a promise: snapshot capture is opt-in and most
+# runs will not have one.
+_SNAPSHOT_HINT = (
+    "If a sandbox snapshot of the original container was captured, that is the "
+    "only remaining route to its state."
+)
+
+
+def _supported_agents_phrase() -> str:
+    names = sorted(CONTINUE_SUPPORTED_AGENTS)
+    return ", ".join(repr(name) for name in names)
+
 
 class RunFolderError(ValueError):
     """Raised when a run folder is missing required artifacts or malformed."""
 
 
+class ContinueUnsupportedError(RunFolderError):
+    """Triage verdict: ``benchflow continue`` cannot resume *this* run.
+
+    Distinct from a failure — nothing went wrong, the run is simply out of
+    scope. Batch mode records these as skips and keeps going (see
+    :mod:`benchflow.continue_run.batch`); the single-run CLI still exits 1.
+    """
+
+    #: stable, machine-readable reason for batch summaries
+    reason_code: str = "continue_unsupported"
+    #: whether a *future* replay ingress could ever rescue this run
+    recoverable_in_principle: bool = False
+
+
+class UnsupportedAgentError(ContinueUnsupportedError):
+    """The run's agent has no replay ingress (see ``CONTINUE_SUPPORTED_AGENTS``)."""
+
+    reason_code = "unsupported_agent"
+
+    def __init__(self, *, agent: str, has_recording: bool) -> None:
+        self.agent = agent
+        self.has_recording = has_recording
+        self.recoverable_in_principle = has_recording
+        self.supported_agents: tuple[str, ...] = tuple(
+            sorted(CONTINUE_SUPPORTED_AGENTS)
+        )
+        used = f"used agent {agent!r}" if agent else "recorded no agent"
+        if has_recording:
+            verdict = (
+                f"This run does have {_TRAJECTORY_RELPATH}, so it becomes "
+                f"continuable as soon as a replay ingress for {agent!r} ships — "
+                "the recording is not the blocker."
+            )
+        else:
+            verdict = _NO_RECORDING_VERDICT
+        super().__init__(
+            f"benchflow continue cannot resume this run: it {used}, and the only "
+            f"supported agent(s) are {_supported_agents_phrase()}. "
+            f"{_WHY_PROTOCOL} {verdict} {_SNAPSHOT_HINT}"
+        )
+
+
+class MissingRecordingError(ContinueUnsupportedError):
+    """No LLM recording exists, so no replay can reconstruct the run."""
+
+    reason_code = "no_llm_recording"
+    recoverable_in_principle = False
+
+    def __init__(self, path: Path) -> None:
+        self.path = path
+        super().__init__(
+            f"missing required artifact: {path} — record-replay needs the LLM "
+            f"trajectory. {_NO_RECORDING_VERDICT} {_SNAPSHOT_HINT}"
+        )
+
+
 @dataclass(frozen=True)
 class RunFolder:
     """Parsed view of an original run's output folder.
@@ -126,6 +220,22 @@ def _read_json(path: Path, *, required: bool) -> dict[str, Any]:
     return data
 
 
+def is_timeout_run(folder: str | Path) -> bool:
+    """Cheap ``result.json``-only check for a timeout/idle-timeout run.
+
+    Triage helper for callers that must classify a folder *after*
+    :func:`load_run_folder` raised — the full load never got far enough to
+    compute :attr:`RunFolder.is_timeout`. Never raises: an unreadable or absent
+    ``result.json`` simply means "not a timeout candidate".
+    """
+    try:
+        result = _read_json(Path(folder).expanduser() / "result.json", required=False)
+    except RunFolderError:
+        return False
+    category = result.get("error_category")
+    return bool(category) and str(category) in _TIMEOUT_CATEGORIES
+
+
 def _load_prompts(path: Path) -> list[str]:
     """Read ``prompts.json`` — a JSON list of strings (or ``{"prompts": [...]}``)."""
     if not path.is_file():
@@ -149,10 +259,7 @@ def load_llm_exchanges(path: Path) -> list[LLMExchange]:
     resume (a single bad record should not strand a recoverable run).
     """
     if not path.is_file():
-        raise RunFolderError(
-            f"missing required artifact: {path} — record-replay needs the LLM "
-            "trajectory. Was this run captured with usage tracking enabled?"
-        )
+        raise MissingRecordingError(path)
     exchanges: list[LLMExchange] = []
     for lineno, raw in enumerate(path.read_text().splitlines(), start=1):
         if not raw.strip():
@@ -183,7 +290,19 @@ def load_run_folder(folder: str | Path, *, require_timeout: bool = False) -> Run
     config = _read_json(path / "config.json", required=True)
     result = _read_json(path / "result.json", required=False)
     prompts = _load_prompts(path / "prompts.json")
-    exchanges = load_llm_exchanges(path / "trajectory" / "llm_trajectory.jsonl")
+
+    # Triage before parsing. The agent gate runs first so an unsupported run is
+    # told whether it is blocked on a missing ingress (recoverable later) or on
+    # a missing recording (never recoverable) — the caller should learn that now,
+    # not after a protocol ingress ships.
+    trajectory_path = path / "trajectory" / "llm_trajectory.jsonl"
+    agent = str(config.get("agent") or result.get("agent") or "")
+    if agent not in CONTINUE_SUPPORTED_AGENTS:
+        raise UnsupportedAgentError(
+            agent=agent, has_recording=trajectory_path.is_file()
+        )
+
+    exchanges = load_llm_exchanges(trajectory_path)
 
     run = RunFolder(
         path=path,
@@ -193,12 +312,6 @@ def load_run_folder(folder: str | Path, *, require_timeout: bool = False) -> Run
         exchanges=exchanges,
     )
 
-    if run.agent != "openhands":
-        raise RunFolderError(
-            f"benchflow continue currently supports the 'openhands' agent only; "
-            f"this run used {run.agent!r}."
-        )
-
     if not run.is_timeout:
         msg = (
             f"run {path.name} has error_category={run.error_category!r}, not a "