fix(observability): let a long task say it is still working

src/hyperloom/agents/robustness/SKILL.md

@@ -126,12 +126,20 @@ host -> subprocess -> envelope -> upstream PolicyGate path.
 | `LLM_MODEL` | no | provider default | RCA model name. With neither override set the chain is openai: `OPENAI_MODEL` → `CODEX_MODEL` → `gpt-5.6-sol`, else anthropic: `ANTHROPIC_MODEL` → `CLAUDE_MODEL` → `claude-opus-5`. |
 | `ROBUSTNESS_LLM_RCA_DISABLED` | no | unset | Set to `1` to forcibly disable the LlmRcaEngine even when credentials are present. |
 
+`Config.discover()` reads the variables above plus the deployment-shape ones
+(`ROBUSTNESS_DISABLE_LOCAL_PROBE`, `ROBUSTNESS_ENABLE_CLUSTER_POD_METRICS`,
+`ROBUSTNESS_NODES`) and nothing else. Every threshold — stall timeouts, disk and
+shm percentages, GPU temperatures — is a field on `Config` with a default in
+`config.py`, changed in code or by whoever constructs the `Config`, not from the
+environment.
+
 ## Symptom -> intent mapping (M1 / M1.5)
 
 | Symptom | Severity | Intents emitted | Source |
 |---------|----------|-----------------|--------|
 | `agent_stall` (≥ stall_timeout_s) | medium | `alert(medium)` | M1 |
 | `agent_stall` (≥ severity_high_after_s) | high | `alert(high)` | M1 |
+| `agent_quiet_work_progressing` (own dispatched work reported within `stall_timeout_s`) | low | `send_message(observation)` | M1.5 |
 | `crash_count_rising` (≥ 2) | medium | `alert(medium)` | M1 |
 | `crash_count_high` (≥ 5) | high | `alert(high)` | M1 |
 | `crash_count_emergency` (≥ 10) | high | `alert(high)` | M1 |
@@ -141,6 +149,7 @@ host -> subprocess -> envelope -> upstream PolicyGate path.
 | `pod_not_running` (other non-Running) | medium | `alert(medium)` | M1 |
 | `pod_no_metrics` (≥ no_metrics_warn_s) | low | `send_message(observation)` | M1 |
 | `local_server_unreachable` (any target down) | medium / high (all down) | `alert(medium)` / `alert(high)` | M1.5 |
+| `local_server_unreachable`, no server process and no benchmark client of this session | — | suppressed (an idle stretch, not an outage) | M1.5 |
 | `log_error_pattern` (CUDA OOM / NCCL / segfault) | high | `alert(high)` | M1.5 |
 | `log_error_pattern` (RuntimeError / generic) | medium | `alert(medium)` | M1.5 |
 | `gpu_thermal_high` (≥ warn_c) | medium | `alert(medium)` | M1.5 |

src/hyperloom/agents/robustness/config.py

@@ -30,6 +30,8 @@
     deepseek_compat_env,
 )
 
+from .sources.local_probe import _OTHER_PROCESS_PATTERNS, _SERVER_PROCESS_PATTERNS
+
 log = logging.getLogger(__name__)
 
 # Primary data source: an optional explicit endpoint (ROBUSTNESS_SERVER_URL),
@@ -94,8 +96,14 @@ def coordinator_db_path(self) -> Path:
         return self.session_dir / "storage" / "coordinator.db"
 
     # -- thresholds --
+    # Set in code, by whoever constructs the Config: :meth:`discover` reads only
+    # the deployment-shape variables listed in SKILL.md, so none of the
+    # thresholds below is settable from the environment.
     gpu_temp_warn_c: float = 85.0
     agent_stall_timeout_s: float = 300.0
+    # Silence past which an agent_stall is HIGH rather than MEDIUM. Deployments
+    # whose work units differ move it; it does not grade a withheld accusation.
+    agent_stall_high_after_s: float = 900.0
 
     # -- LLM for RCA (auto-detected from Claw sandbox env) --
     llm_model: str = "claude-opus-5"
@@ -279,33 +287,18 @@ def coordinator_db_path(self) -> Path:
     state_store_enabled: bool = True
 
     # -- server process patterns --
-    # Mirrors ``local_probe._DEFAULT_PROCESS_PATTERNS`` so the
-    # gpu_memory_leaked "no live owner" check matches every legitimate VRAM
-    # holder.
+    # Defaulted from the probe's own lists rather than restated here: a
+    # framework added to one copy but not the other used to appear as a matched
+    # process that is not a server, which silently disabled
+    # ``local_server_unreachable``. ``server_process_patterns`` is what a health
+    # probe may hold accountable for answering a port; the benchmark list adds
+    # the other legitimate VRAM holders the gpu_memory_leaked "no live owner"
+    # check has to see.
     server_process_patterns: list[str] = field(
-        default_factory=lambda: [
-            # SGLang
-            "sglang.srt",
-            "sglang.launch_server",
-            # vLLM
-            "vllm.entrypoints",
-            "vllm serve",
-            "vllm.v1.engine.core",
-            "vllm.engine.async_llm_engine",
-            "EngineCore",
-            # Magpie / InferenceX
-            "Magpie",
-            "inferencex",
-            # Ray + JIT compilation
-            "ray::IDLE",
-            "raylet",
-            "hipcc",
-        ]
+        default_factory=lambda: list(_SERVER_PROCESS_PATTERNS),
     )
     benchmark_process_patterns: list[str] = field(
-        default_factory=lambda: [
-            "benchmark_serving",
-        ]
+        default_factory=lambda: list(_OTHER_PROCESS_PATTERNS),
     )
 
     @classmethod

src/hyperloom/agents/robustness/factory.py

@@ -93,6 +93,68 @@ async def aclose(self) -> None:
             await self.server_client.aclose()
 
 
+def _build_local_probe_config(config: Config) -> LocalProbeConfig:
+    """Project the agent config onto the local probe's own configuration.
+
+    ``server_process_patterns`` is passed through as both the server subset and
+    part of the match list, so a framework an operator adds to that knob is
+    matched *and* recognised as a server. Splitting those two decisions across
+    separately-maintained lists is what let a configured framework show up as
+    ``is_server=False`` and silently disable ``local_server_unreachable``.
+
+    Args:
+        config (Config): The discovered agent configuration.
+
+    Returns:
+        LocalProbeConfig: Configuration for :class:`LocalProbeSource`.
+    """
+    # Auto-include the local inference server health endpoint.
+    probe_targets = list(config.health_probe_targets)
+    if (
+        config.auto_probe_inference_server
+        and config.inference_server_health_url
+        and config.inference_server_health_url not in probe_targets
+    ):
+        probe_targets.append(config.inference_server_health_url)
+
+    extra_log_globs: tuple[str, ...] = (
+        "runs/*/*/server.log",
+        "runs/*/*/server_log",
+        "runs/*/server.log",
+        "runs/*/*/*/server.log",
+        "runs/*/*/*/*/server.log",
+    )
+    if config.server_log_extra_globs:
+        extra_log_globs = (
+            *extra_log_globs,
+            *(g.strip() for g in config.server_log_extra_globs.split(":") if g.strip()),
+        )
+
+    return LocalProbeConfig(
+        session_dir=config.session_dir,
+        process_patterns=tuple(config.server_process_patterns + config.benchmark_process_patterns),
+        server_process_patterns=tuple(config.server_process_patterns),
+        health_probe_targets=tuple(probe_targets),
+        health_probe_timeout_s=config.health_probe_timeout_s,
+        ray_probe_enabled=config.ray_probe_enabled,
+        ray_probe_timeout_s=config.ray_probe_timeout_s,
+        fd_probe_enabled=config.fd_probe_enabled,
+        fd_probe_pid=config.fd_probe_pid,
+        decision_audit_enabled=config.decision_audit_enabled,
+        decision_audit_max_integrate=config.decision_audit_max_integrate,
+        decision_audit_max_oob_attempts=(config.decision_audit_max_oob_attempts),
+        preflight_enabled=config.preflight_enabled,
+        critic_health_enabled=config.critic_health_enabled,
+        max_critic_judge_bundles=config.critic_health_max_judge_bundles,
+        extra_server_log_globs=extra_log_globs,
+        max_extra_server_logs=config.server_log_max_extra,
+        state_integrity_enabled=config.state_integrity_enabled,
+        external_deps_enabled=config.external_deps_enabled,
+        external_mount_stat_timeout_s=(config.external_mount_stat_timeout_s),
+        external_gateway_probe_url=config.external_gateway_probe_url,
+    )
+
+
 def build_reactor_components(
     config: Config,
     *,
@@ -138,28 +200,6 @@ def build_reactor_components(
             "config.robustness_server_url is empty",
         )
 
-    # Auto-include the local inference server health endpoint.
-    probe_targets = list(config.health_probe_targets)
-    if (
-        config.auto_probe_inference_server
-        and config.inference_server_health_url
-        and config.inference_server_health_url not in probe_targets
-    ):
-        probe_targets.append(config.inference_server_health_url)
-
-    extra_log_globs: tuple[str, ...] = (
-        "runs/*/*/server.log",
-        "runs/*/*/server_log",
-        "runs/*/server.log",
-        "runs/*/*/*/server.log",
-        "runs/*/*/*/*/server.log",
-    )
-    if config.server_log_extra_globs:
-        extra_log_globs = (
-            *extra_log_globs,
-            *(g.strip() for g in config.server_log_extra_globs.split(":") if g.strip()),
-        )
-
     # Multi-node guard: ``disable_local_probe`` swaps LocalProbe for a quiet stub.
     fallback: Source
     if config.disable_local_probe:
@@ -168,30 +208,7 @@ def build_reactor_components(
             reason="config.disable_local_probe is True (multi-node policy)",
         )
     else:
-        fallback = LocalProbeSource(
-            LocalProbeConfig(
-                session_dir=config.session_dir,
-                process_patterns=tuple(config.server_process_patterns + config.benchmark_process_patterns),
-                health_probe_targets=tuple(probe_targets),
-                health_probe_timeout_s=config.health_probe_timeout_s,
-                ray_probe_enabled=config.ray_probe_enabled,
-                ray_probe_timeout_s=config.ray_probe_timeout_s,
-                fd_probe_enabled=config.fd_probe_enabled,
-                fd_probe_pid=config.fd_probe_pid,
-                decision_audit_enabled=config.decision_audit_enabled,
-                decision_audit_max_integrate=config.decision_audit_max_integrate,
-                decision_audit_max_oob_attempts=(config.decision_audit_max_oob_attempts),
-                preflight_enabled=config.preflight_enabled,
-                critic_health_enabled=config.critic_health_enabled,
-                max_critic_judge_bundles=config.critic_health_max_judge_bundles,
-                extra_server_log_globs=extra_log_globs,
-                max_extra_server_logs=config.server_log_max_extra,
-                state_integrity_enabled=config.state_integrity_enabled,
-                external_deps_enabled=config.external_deps_enabled,
-                external_mount_stat_timeout_s=(config.external_mount_stat_timeout_s),
-                external_gateway_probe_url=config.external_gateway_probe_url,
-            )
-        )
+        fallback = LocalProbeSource(_build_local_probe_config(config))
 
     router = DegradeRouter(
         primary,
@@ -211,6 +228,7 @@ def build_reactor_components(
     signal_configs: dict[str, Any] = {
         "stall": StallConfig(
             stall_timeout_s=config.agent_stall_timeout_s,
+            severity_high_after_s=config.agent_stall_high_after_s,
         ),
         "crash": CrashConfig(),
         "event": EventConfig(
@@ -225,6 +243,7 @@ def build_reactor_components(
             shm_used_crit_pct=config.shm_used_crit_pct,
             fd_warn_used_pct=config.fd_warn_used_pct,
             fd_crit_used_pct=config.fd_crit_used_pct,
+            session_dir=config.session_dir,
         ),
         "gpu_leak": GpuLeakConfig(
             util_mem_pct_threshold=config.gpu_leak_util_mem_pct_threshold,
@@ -498,6 +517,9 @@ async def fetch(self, ctx: Any) -> SourceData:  # noqa: ARG002 - protocol
         """
         return SourceData(
             degraded_reason=f"local-probe disabled: {self.reason}",
+            # Nothing looked: an empty process list here is ignorance, not
+            # evidence that no server is running.
+            local_processes_known=False,
             sources_used=[self.name],
         )
 

src/hyperloom/agents/robustness/prompts/rca.md

@@ -26,7 +26,7 @@ Symptoms come from these signal families; knowing the family helps you interpret
 | H — Time / budget | `budget_strategy_drift`, `budget_burn_no_gain`, `deadline_imminent`, `deadline_warning`, `deadline_hard_cutoff` |
 | I — State integrity | `state_json_corrupt`, `coordinator_wal_bloat`, `stale_lease`, `inbox_bloat`, `coordinator_zombie` |
 | J — External deps | `gateway_auth_outage`, `wekafs_degraded`, `tracelens_cli_missing` |
-| baseline / stall | `agent_stall`, `crash_count_rising`, `crash_count_high`, `crash_count_emergency`, `repeated_policy_denied`, `repeated_failure`, `recover_unsuccessful`, `cluster_fault` |
+| baseline / stall | `agent_stall`, `agent_quiet_work_progressing`, `crash_count_rising`, `crash_count_high`, `crash_count_emergency`, `repeated_policy_denied`, `repeated_failure`, `recover_unsuccessful`, `cluster_fault` |
 
 ## Output contract