feat(debug-trace-server): age-based witness routing, configurable old-block budget, tip buffer

AGENTS.md

@@ -107,6 +107,8 @@ Two operating modes:
 - **Local cache mode** — With `data_dir`, enables chain sync to pre-fetch blocks into `ValidatorDB` for faster serving.
 
 The server includes an HTTP response cache (`quick_cache`) for pre-serialized JSON and a `DataProvider` with single-flight request coalescing.
+In local cache mode with two or more witness endpoints, request-serving witness fetches route by block age: blocks at least `--witness-local-window` blocks below the local tip skip the first witness endpoint (the internal generator, which prunes beyond its `BACKUP` window) and fetch from the remaining endpoints.
+The background chain-sync prefetch always uses the full endpoint chain — it fetches at the sync frontier, which stays within the generator's retention unless `--blocks-to-keep` exceeds that retention during deep catch-up.
 
 ### Key Source Files
 

README.md

@@ -98,6 +98,16 @@ Two operating modes:
 - **Stateless mode** (no `--data-dir`): All data fetched from remote RPC on demand.
 - **Local cache mode** (with `--data-dir`): Enables chain sync to pre-fetch blocks for faster serving.
 
+**Witness endpoint ordering:**
+For the debug-trace-server, the first `--witness-endpoint` is positionally special: it is treated as the internal witness generator, so list the generator first and durable fallbacks (e.g. an R2-backed witness service) after it.
+In local cache mode with two or more witness endpoints, requests for blocks at least `--witness-local-window` blocks below the local tip skip that first endpoint and fetch from the remaining endpoints, because the generator only retains a recent window (its `BACKUP`, deployed at 4096) and probing it for pruned blocks is a guaranteed miss.
+The background chain-sync prefetch always uses the full endpoint chain.
+
+**Witness routing and sync knobs** (each also settable via its `DEBUG_TRACE_SERVER_*` env var):
+- `--witness-local-window`: Block-age threshold for the historical witness route (default: 4096; should match the generator's `BACKUP`).
+- `--witness-old-block-timeout`: Witness-stage budget in seconds for blocks at or below the local tip (defaults to the full `--witness-timeout` budget, tracking it when raised; lower it to fail fast on pruned blocks).
+- `--tip-buffer`: Stay this many blocks behind the upstream head during chain sync so fetches don't race the witness generator (default: 2; must be smaller than `--blocks-to-keep`).
+
 ### Environment Variables
 
 Each command-line flag has an equivalent environment variable:

bin/debug-trace-server/src/chain_sync.rs

@@ -24,6 +24,12 @@ use crate::{metrics, response_cache::ResponseCache, server_db::BlockStore};
 /// Witnesses go through the zero-validation light decode (`get_witness_light`):
 /// the server never verifies the proof, so the full decode's per-point
 /// elliptic-curve work bought nothing.
+///
+/// Witness fetches deliberately use the full endpoint chain (no age-based routing): the sync
+/// frontier trails the remote head by only `tip_buffer`, inside the generator's retention —
+/// except during deep catch-up with `--blocks-to-keep` beyond that retention, where each
+/// pruned block burns one generator probe before failover (accepted; routing here would need
+/// a remote-head anchor instead of the local tip).
 pub struct TraceFetcher {
     pub rpc_client: Arc<RpcClient>,
 }

bin/debug-trace-server/src/main.rs

@@ -3,6 +3,9 @@
 //! # Overview
 //! A standalone RPC server for `debug_*` and `trace_*` methods using stateless execution.
 //! Data can be fetched from upstream RPC endpoints or from a local database with chain sync.
+//! Request-serving witness fetches route by block age: historical blocks skip the internal
+//! generator endpoint (which only retains a small recent window) and go straight to the
+//! fallback endpoints. Chain-sync prefetch always uses the full chain.
 //!
 //! # Architecture
 //! ```text
@@ -25,6 +28,8 @@
 //! ┌─────────────────────────────────────────────────────────────────┐
 //! │                      DataProvider                               │
 //! │  Multi-level lookup: Local DB → Remote RPC (with single-flight) │
+//! │  Witnesses: near-tip → full endpoint chain; historical skips    │
+//! │  the internal generator (guaranteed miss) → fallback endpoints  │
 //! └─────────────────────────────────────────────────────────────────┘
 //! ```
 //!
@@ -37,7 +42,7 @@
 //! - `debug_getCacheStatus` - Query current response cache status
 //!
 //! # Operating Modes
-//! - **Stateless mode**: Without `data_dir`, all data is fetched from remote RPC
+//! - **Stateless mode**: Without `data_dir`, all data is fetched from remote RPC endpoints
 //! - **Local cache mode**: With `data_dir`, enables chain sync to pre-fetch blocks into local DB
 
 use std::{path::PathBuf, sync::Arc};
@@ -68,7 +73,7 @@ mod server_db;
 mod timing;
 mod tracing_executor;
 
-use data_provider::{DataProvider, NoopContractStore};
+use data_provider::{DataProvider, NoopContractStore, WitnessFetchConfig};
 use response_cache::{DEFAULT_RESPONSE_CACHE_ESTIMATED_ITEMS, ResponseCache, ResponseCacheConfig};
 use rpc_service::RpcContext;
 use server_db::{BlockStore, ServerDB};
@@ -97,7 +102,9 @@ struct Args {
 
     /// One or more upstream witness endpoint URLs for fetching witness data (tried in order).
     /// Accepts repeated flags (`--witness-endpoint a --witness-endpoint b`) or a comma-separated
-    /// list (`--witness-endpoint a,b`, also via the env var).
+    /// list (`--witness-endpoint a,b`, also via the env var). The first endpoint is positionally
+    /// special: it is treated as the internal witness generator and is skipped for historical
+    /// blocks (see `--witness-local-window`).
     #[clap(
         long,
         env = "DEBUG_TRACE_SERVER_WITNESS_ENDPOINT",
@@ -198,10 +205,38 @@ struct Args {
     data_max_concurrent_requests: Option<usize>,
 
     /// Maximum concurrent in-flight witness fetches, independent of the data cap.
-    /// Omit for unlimited.
+    /// Omit for unlimited. One global cap: recent and historical witness routes share it.
     #[clap(long, env = "DEBUG_TRACE_SERVER_WITNESS_MAX_CONCURRENT_REQUESTS")]
     witness_max_concurrent_requests: Option<usize>,
 
+    /// Blocks fewer than this many blocks below the local tip fetch witnesses through the
+    /// full witness endpoint chain (internal generator first); blocks at least this far below
+    /// skip the first witness endpoint — the generator only retains about this window (its
+    /// `BACKUP` env, deployed at 4096), so probing it for historical blocks is a guaranteed
+    /// miss. Requires at least two witness endpoints and a local DB (`--data-dir`), whose tip
+    /// anchors block age; otherwise all blocks use the full chain. Applies to request
+    /// serving; chain-sync prefetch always uses the full chain. Should match the generator's
+    /// `BACKUP`.
+    #[clap(
+        long,
+        env = "DEBUG_TRACE_SERVER_WITNESS_LOCAL_WINDOW",
+        default_value_t = data_provider::DEFAULT_WITNESS_LOCAL_WINDOW
+    )]
+    witness_local_window: u64,
+
+    /// Witness-stage budget in seconds for blocks at or below the local tip. Defaults to the
+    /// full `--witness-timeout` budget (tracking it when raised); lower it to fail fast on
+    /// blocks whose witness is likely pruned everywhere. Clamped to `--witness-timeout`.
+    #[clap(long, env = "DEBUG_TRACE_SERVER_WITNESS_OLD_BLOCK_TIMEOUT")]
+    witness_old_block_timeout: Option<u64>,
+
+    /// Chain-sync pipeline tip buffer: stay this many blocks behind the upstream head so the
+    /// fetcher does not race the witness generator — a fetch issued the moment a block appears
+    /// typically arrives before its witness is written and burns a failed round plus a backoff
+    /// sleep. 0 = fetch right at the head.
+    #[clap(long, env = "DEBUG_TRACE_SERVER_TIP_BUFFER", default_value_t = 2)]
+    tip_buffer: u64,
+
     /// Per-attempt RPC timeout (milliseconds). Must be ≥ 100ms.
     #[clap(
         long,
@@ -255,9 +290,33 @@ fn parse_size(s: &str) -> Result<u64, String> {
     value.checked_mul(multiplier).ok_or_else(|| format!("size overflow: '{}'", s))
 }
 
+/// Effective old-block witness budget in seconds: the flag when set (clamped to
+/// `--witness-timeout`), otherwise the full `--witness-timeout` budget — raising the witness
+/// budget also raises the old-block cap.
+fn old_block_witness_timeout_secs(args: &Args) -> u64 {
+    args.witness_old_block_timeout.unwrap_or(args.witness_timeout).min(args.witness_timeout)
+}
+
+/// Validates cross-flag invariants that clap cannot express per-field.
+fn validate_args(args: &Args) -> Result<()> {
+    // Only meaningful with chain sync: the fetcher holds the local tip `tip_buffer` blocks
+    // behind the head, and `blocks_to_keep` doubles as the stale-reset threshold. A buffer at
+    // or past the threshold makes the built-in lag itself look stale, so every transient
+    // restart would reset the anchor and leave gaps in stored history.
+    if args.data_dir.is_some() && args.tip_buffer >= args.blocks_to_keep {
+        eyre::bail!(
+            "--tip-buffer ({}) must be smaller than --blocks-to-keep ({})",
+            args.tip_buffer,
+            args.blocks_to_keep
+        );
+    }
+    Ok(())
+}
+
 #[tokio::main]
 async fn main() -> Result<()> {
     let args = Args::parse();
+    validate_args(&args)?;
     let _log_guard = args.log.init_tracing()?;
 
     info!(
@@ -269,6 +328,9 @@ async fn main() -> Result<()> {
         rpc_endpoints = ?args.rpc_endpoint,
         witness_endpoints = ?args.witness_endpoint,
         witness_timeout_secs = args.witness_timeout,
+        witness_old_block_timeout_secs = old_block_witness_timeout_secs(&args),
+        witness_local_window = args.witness_local_window,
+        tip_buffer = args.tip_buffer,
         response_cache_disabled,
         response_cache_max_size = args.response_cache_max_size,
         response_cache_estimated_items = args.response_cache_estimated_items,
@@ -306,6 +368,24 @@ async fn main() -> Result<()> {
     .with_metrics(Arc::new(metrics::TraceRpcMetrics));
     let rpc_client =
         Arc::new(RpcClient::new_with_config(&data_apis, &witness_apis, rpc_config, None)?);
+
+    match (witness_apis.len() >= 2, args.data_dir.is_some()) {
+        (true, true) => info!(
+            witness_local_window = args.witness_local_window,
+            // The credential-stripped label, not the raw URL — configured endpoint URLs may
+            // carry userinfo or token queries and this log line is info-level.
+            skipped_endpoint = rpc_client.witness_provider_label(0).unwrap_or("<none>"),
+            "Historical witnesses (at least the local window below the tip) skip the internal \
+             generator endpoint"
+        ),
+        (true, false) => warn!(
+            "Multiple witness endpoints but no --data-dir: historical witness routing is \
+             inactive (block age is anchored to the local DB tip); all witness fetches use the \
+             full endpoint chain"
+        ),
+        (false, _) => debug!("Single witness endpoint configured; no historical witness routing"),
+    }
+
     let validator_db = init_validator_db(&args, &rpc_client).await?;
 
     // Keep concrete ServerDB for pipeline (needs Sized), and dyn BlockStore for data_provider
@@ -322,12 +402,19 @@ async fn main() -> Result<()> {
     };
     let contract_cache = Arc::new(ContractCache::new(contract_store));
 
+    let witness_cfg = WitnessFetchConfig {
+        witness_timeout: std::time::Duration::from_secs(args.witness_timeout),
+        old_block_witness_timeout: std::time::Duration::from_secs(old_block_witness_timeout_secs(
+            &args,
+        )),
+        local_window: args.witness_local_window,
+    };
     let data_provider = Arc::new(DataProvider::new(
         rpc_client.clone(),
         block_store.clone(),
         contract_cache,
-        args.witness_timeout,
-        args.block_fetch_timeout,
+        witness_cfg,
+        std::time::Duration::from_secs(args.block_fetch_timeout),
     ));
 
     let chain_spec = load_chain_spec(&args)?;
@@ -358,6 +445,7 @@ async fn main() -> Result<()> {
         // the crate boundary; mutate a default instance instead.
         let mut pipeline_cfg = PipelineConfig::default();
         pipeline_cfg.concurrent_workers = 1;
+        pipeline_cfg.tip_buffer = args.tip_buffer;
         pipeline_cfg.stale_reset_threshold = Some(args.blocks_to_keep);
         let config = Arc::new(pipeline_cfg);
         let processor = Arc::new(TraceProcessor);
@@ -671,6 +759,84 @@ mod tests {
         );
     }
 
+    /// Pins the tiered-witness-routing knob defaults (`--witness-local-window` must track the
+    /// generator's `BACKUP`, the old-block budget defaults to the full witness budget) and the
+    /// CLI + env parsing of all three knobs — a typo in an env attribute string would
+    /// otherwise ship silently to env-only container deployments.
+    /// Parses `Args` from the minimal required flags plus `extra`. Callers must hold
+    /// `stateless_test_utils::env::env_lock()` — parsing reads `DEBUG_TRACE_SERVER_*` env
+    /// vars, so it must be serialized with the tests that mutate them.
+    fn parse_args(extra: &[&str]) -> Args {
+        let base =
+            ["debug-trace-server", "--rpc-endpoint", "http://r", "--witness-endpoint", "http://w"];
+        Args::try_parse_from(base.iter().chain(extra)).unwrap()
+    }
+
+    #[test]
+    fn tiered_routing_flag_defaults() {
+        let guard = stateless_test_utils::env::env_lock();
+
+        let defaults = parse_args(&[]);
+        assert_eq!(defaults.witness_local_window, data_provider::DEFAULT_WITNESS_LOCAL_WINDOW);
+        assert_eq!(defaults.witness_old_block_timeout, None);
+        assert_eq!(
+            old_block_witness_timeout_secs(&defaults),
+            data_provider::DEFAULT_WITNESS_TIMEOUT_SECS,
+            "old blocks default to the full witness budget",
+        );
+        assert_eq!(defaults.tip_buffer, 2);
+
+        // The unset default tracks a raised --witness-timeout; an explicit flag wins.
+        assert_eq!(old_block_witness_timeout_secs(&parse_args(&["--witness-timeout", "20"])), 20);
+        assert_eq!(
+            old_block_witness_timeout_secs(&parse_args(&[
+                "--witness-timeout",
+                "20",
+                "--witness-old-block-timeout",
+                "3"
+            ])),
+            3
+        );
+
+        assert_eq!(parse_args(&["--tip-buffer", "0"]).tip_buffer, 0);
+        assert_eq!(parse_args(&["--witness-local-window", "128"]).witness_local_window, 128);
+        assert_eq!(
+            parse_args(&["--witness-old-block-timeout", "3"]).witness_old_block_timeout,
+            Some(3)
+        );
+
+        let env = |name, value: &str| {
+            stateless_test_utils::env::with_env_var(&guard, name, value, || parse_args(&[]))
+        };
+        assert_eq!(env("DEBUG_TRACE_SERVER_TIP_BUFFER", "5").tip_buffer, 5);
+        assert_eq!(env("DEBUG_TRACE_SERVER_WITNESS_LOCAL_WINDOW", "256").witness_local_window, 256);
+        assert_eq!(
+            env("DEBUG_TRACE_SERVER_WITNESS_OLD_BLOCK_TIMEOUT", "4").witness_old_block_timeout,
+            Some(4)
+        );
+    }
+
+    /// `--tip-buffer` must stay below `--blocks-to-keep` when chain sync is enabled: the
+    /// pipeline's built-in lag would otherwise satisfy the stale-reset test on every
+    /// transient restart. Inert in stateless mode, where neither flag is used.
+    #[test]
+    fn tip_buffer_must_stay_below_blocks_to_keep() {
+        let _guard = stateless_test_utils::env::env_lock();
+
+        // Stateless mode (no data dir): both flags are inert, any combination is accepted.
+        assert!(validate_args(&parse_args(&["--tip-buffer", "2000"])).is_ok());
+
+        let with_db = |extra: &[&str]| {
+            let mut v = vec!["--data-dir", "/tmp/x"];
+            v.extend_from_slice(extra);
+            parse_args(&v)
+        };
+        assert!(validate_args(&with_db(&[])).is_ok());
+        assert!(validate_args(&with_db(&["--tip-buffer", "999"])).is_ok());
+        assert!(validate_args(&with_db(&["--tip-buffer", "1000"])).is_err());
+        assert!(validate_args(&with_db(&["--tip-buffer", "5", "--blocks-to-keep", "5"])).is_err());
+    }
+
     /// Verifies a concurrency cap flag parses via CLI and env var, and defaults to `None`.
     fn assert_concurrency_flag(
         flag: &str,

bin/debug-trace-server/src/metrics.rs

@@ -206,7 +206,10 @@ impl CacheMetrics {
     }
 }
 
-/// Tracks which source provided block data (cache/db/witness_generator).
+/// Tracks which source provided block data. Sources: `cache`, `db`, and the two RPC witness
+/// routes — `witness_generator` (full endpoint chain, generator first) and
+/// `witness_historical` (skip-generator chain for blocks at least the local window below the
+/// tip). The RPC path as a whole is the sum of the two witness labels.
 #[derive(Clone, Metrics)]
 #[metrics(scope = "debug_trace")]
 pub struct DataSourceMetrics {
@@ -421,6 +424,7 @@ fn pre_register_all_metrics() {
     let _ = DataSourceMetrics::new_for_source("cache");
     let _ = DataSourceMetrics::new_for_source("db");
     let _ = DataSourceMetrics::new_for_source("witness_generator");
+    let _ = DataSourceMetrics::new_for_source("witness_historical");
 
     // Data Fetch Layer: single-flight
     let _ = SingleFlightMetrics::new_for_type("new");
@@ -439,6 +443,7 @@ fn pre_register_all_metrics() {
 
     // Witness Layer
     let _ = WitnessSourceMetrics::new_for_source("witness_generator");
+    let _ = WitnessSourceMetrics::new_for_source("witness_historical");
 
     // Execution Layer (per method)
     let _ = EvmExecutionMetrics::new_for_method(METHOD_DEBUG_TRACE_BLOCK_BY_NUMBER);