fix(continuous-sync): require evidence before halting stalled nodes

deploy/continuous-sync/README.md

@@ -39,6 +39,33 @@ v2 stack.
 The same commit may be tested repeatedly. That is intentional: the fleet is a
 continuous sync canary, not a once-per-SHA CI job.
 
+For Zakura and dual-stack nodes, the controller distinguishes an idle chain from
+a stalled node. It compares the exact committed block height with the exact
+local header-chain height. A header height at or below the committed height
+means that the node has no local header backlog, so a long Mainnet block
+interval does not start the stall deadline. A higher header height starts the
+deadline because the node has a local backlog that it can process. New
+committed progress restarts the deadline.
+
+A dual-stack node that hands block sync to legacy fallback stops the run
+immediately, naming the handoff as the reason. The canary exists to catch v2
+stalls before legacy masks them, so an active fallback is the failure it is
+looking for, not a recovery window to wait out. Measuring legacy progress after
+the handoff would report a healthy node while the v2 stack stayed stalled.
+
+The controller records a metrics error or missing exact height as unavailable
+status evidence. It does not classify that sample as a sync stall. Continuous
+status unavailability has its own deadline and failure reason. Wall-clock tip
+estimates remain available for diagnostics, but they do not supply stall
+evidence.
+
+The legacy-only node does not run the Zakura header chain. It retains its
+1800-second height-only deadline. The cluster monitor independently requires a
+healthy peer with an exact committed height to advance before it reports a local
+sync stall. It never uses a wall-clock estimate as peer evidence. That peer
+evidence covers a header-sync failure where the local node never learns the
+newer header.
+
 ## Failure Semantics
 
 Any build, install, cleanup, startup, sync, stall, timeout, disk, metrics, or

deploy/continuous-sync/alert-monitor.py

@@ -299,16 +299,17 @@ def update_progress_state(state: dict[str, Any], statuses: list[dict[str, Any]],
             record["controller_run"] = run_id
 
         height = status.get("height")
-        if height is None:
+        if height is None or status.get("height_is_exact") is not True:
             continue
         previous_height = record.get("height")
         record["height"] = height
-        if (
-            record.pop("progress_reset_pending", False)
-            or previous_height is None
-            or height > previous_height
-        ):
+        reset = record.pop("progress_reset_pending", False)
+        if reset or previous_height is None:
             record["last_progress"] = ts
+            record.pop("last_advance", None)
+        elif height > previous_height:
+            record["last_progress"] = ts
+            record["last_advance"] = ts
         else:
             record.setdefault("last_progress", ts)
 
@@ -459,16 +460,24 @@ def run_once(config: dict[str, Any]) -> int:
     last_progress = int(record.get("last_progress", ts))
     age = ts - last_progress
     peer_evidence = []
-    if node_healthy(local_status) and height is not None:
+    if (
+        node_healthy(local_status)
+        and height is not None
+        and local_status.get("height_is_exact") is True
+    ):
         for peer in statuses:
-            if peer["hostname"] == local_host or not node_healthy(peer):
+            if (
+                peer["hostname"] == local_host
+                or not node_healthy(peer)
+                or peer.get("height_is_exact") is not True
+            ):
                 continue
             peer_height = peer.get("height")
             peer_record = state.get("nodes", {}).get(peer["hostname"], {})
             if (
                 peer_height is not None
                 and peer_height > height
-                and int(peer_record.get("last_progress", 0)) > last_progress
+                and int(peer_record.get("last_advance", 0)) > last_progress
             ):
                 peer_evidence.append(f"{peer['hostname']} advanced to height {peer_height}")
     stalled = (
@@ -490,6 +499,7 @@ def run_once(config: dict[str, Any]) -> int:
     local_progressed = (
         isinstance(height, int)
         and isinstance(previous_local_height, int)
+        and local_status.get("height_is_exact") is True
         and height > previous_local_height
     )
     if controller_owns_lifecycle:

deploy/continuous-sync/alert-status.py

@@ -52,23 +52,20 @@ def service_active(service: str) -> bool:
     return active_state in {"active", "reloading", "refreshing"}
 
 
-def metric_height(text: str) -> int | None:
-    # Prefer finalized/verified block progress over header-only metrics.
-    priority = [
+def metric_height_observation(text: str) -> tuple[int | None, str | None]:
+    # Finalized and verifier-only gauges can trail or lead the best committed
+    # tip, so they cannot prove node progress.
+    exact = [
         "state_memory_best_committed_block_height",
         "state_memory_committed_block_height",
-        "state_finalized_block_height",
-        "state_checkpoint_finalized_block_height",
         "zcash_chain_verified_block_height",
         "sync_block_verified_tip_height",
-        "checkpoint_verified_height",
-        "checkpoint_processing_next_height",
     ]
     estimated = [
         "sync_estimated_network_tip_height",
         "sync_estimated_distance_to_tip",
     ]
-    values = {name: [] for name in priority + estimated}
+    values = {name: [] for name in exact + estimated}
     for line in text.splitlines():
         if not line or line.startswith("#"):
             continue
@@ -83,15 +80,19 @@ def metric_height(text: str) -> int | None:
             values[dotted_base].append(int(float(parts[1])))
         except ValueError:
             continue
-    for name in priority:
+    for name in exact:
         if values[name]:
-            return max(values[name])
+            return max(values[name]), name
     if all(values[name] for name in estimated):
         tip = max(values["sync_estimated_network_tip_height"])
         distance = min(values["sync_estimated_distance_to_tip"])
         if 0 <= distance <= tip:
-            return tip - distance
-    return None
+            return tip - distance, "estimated_tip_minus_distance"
+    return None, None
+
+
+def metric_height(text: str) -> int | None:
+    return metric_height_observation(text)[0]
 
 
 def node_info(config: dict[str, Any], hostname: str) -> dict[str, Any]:
@@ -131,11 +132,12 @@ def status(config: dict[str, Any]) -> dict[str, Any]:
 
     metrics_status = "unavailable"
     height = None
+    height_source = None
     try:
         with urllib.request.urlopen(metrics_url, timeout=METRICS_TIMEOUT_SECONDS) as response:
             metrics = response.read().decode("utf-8", "replace")
         metrics_status = "ok"
-        height = metric_height(metrics)
+        height, height_source = metric_height_observation(metrics)
     except Exception as exc:
         metrics_status = f"unavailable: {type(exc).__name__}"
 
@@ -147,6 +149,8 @@ def status(config: dict[str, Any]) -> dict[str, Any]:
         "service_active": service_active(service),
         "metrics_status": metrics_status,
         "height": height,
+        "height_source": height_source,
+        "height_is_exact": height_source not in (None, "estimated_tip_minus_distance"),
         "controller_state": controller_state(controller_state_path),
         "connection": node.get("ssh_string", f"root@{node.get('public_ip', 'unknown')}"),
         "alias_connection": f"ssh {node.get('alias', hostname)}",