From a385a21e9ed6f321d416912149c0a064fa8d334e Mon Sep 17 00:00:00 2001 From: Haiyue Chen Date: Mon, 27 Jul 2026 18:34:52 +0200 Subject: [PATCH] feat(node): surface healthy provider counts on /debug/node_config The startup probe publishes its per-chain healthy-provider counts over a watch channel; the web server overlays them onto `/debug/node_config` as `foreign_chains_provider_health` (empty until the probe completes, always present, mirroring `foreign_chains_provider_counts`). --- crates/e2e-tests/src/cluster.rs | 16 +++++++ .../tests/startup_foreign_chain_health.rs | 18 +++++++ crates/node/src/foreign_chain_health.rs | 46 +++++++++++++----- crates/node/src/run.rs | 4 ++ crates/node/src/tests.rs | 3 ++ crates/node/src/web.rs | 48 +++++++++++++++++-- 6 files changed, 120 insertions(+), 15 deletions(-) diff --git a/crates/e2e-tests/src/cluster.rs b/crates/e2e-tests/src/cluster.rs index 0b8ee8d4e..f14fee195 100644 --- a/crates/e2e-tests/src/cluster.rs +++ b/crates/e2e-tests/src/cluster.rs @@ -773,6 +773,22 @@ impl MpcCluster { }) } + /// Fetches a node's `/debug/node_config` and returns the parsed JSON. + pub async fn fetch_node_config(&self, node_index: usize) -> anyhow::Result { + let web_address = match &self.nodes[node_index] { + MpcNodeState::Running(n) => n.web_address(), + MpcNodeState::Stopped(_) => anyhow::bail!("node {node_index} is not running"), + }; + let url = format!("http://{web_address}/debug/node_config"); + let body = reqwest::get(&url) + .await + .context("GET /debug/node_config failed")? + .text() + .await + .context("failed to read /debug/node_config body")?; + serde_json::from_str(&body).context("failed to parse /debug/node_config JSON") + } + pub fn wipe_db(&self, indices: &[usize]) -> anyhow::Result<()> { for &idx in indices { match &self.nodes[idx] { diff --git a/crates/e2e-tests/tests/startup_foreign_chain_health.rs b/crates/e2e-tests/tests/startup_foreign_chain_health.rs index 52e92f64a..777b55798 100644 --- a/crates/e2e-tests/tests/startup_foreign_chain_health.rs +++ b/crates/e2e-tests/tests/startup_foreign_chain_health.rs @@ -165,5 +165,23 @@ async fn startup_health_check__should_probe_providers_by_chain_identity() { "healthy: node 0 passes 1 of 2, node 1 fails its identity-less base check (0), node 2 none" ); + // Then the same healthy counts surface on `/debug/node_config`: node 0 overlays its probed + // `base` count, node 1 (no identity) reports 0, and node 2 (nothing configured) is empty. + assert_eq!( + cluster.fetch_node_config(0).await.unwrap()["foreign_chains_provider_health"], + serde_json::json!({ "base": 1 }), + "node 0 must expose its healthy `base` count on /debug/node_config" + ); + assert_eq!( + cluster.fetch_node_config(1).await.unwrap()["foreign_chains_provider_health"], + serde_json::json!({ "base": 0 }), + "node 1's identity-less `base` check fails, so its healthy count is 0" + ); + assert_eq!( + cluster.fetch_node_config(2).await.unwrap()["foreign_chains_provider_health"], + serde_json::json!({}), + "node 2 configured nothing, so its health map is empty" + ); + drop(cluster); } diff --git a/crates/node/src/foreign_chain_health.rs b/crates/node/src/foreign_chain_health.rs index d26629fd3..588a084cb 100644 --- a/crates/node/src/foreign_chain_health.rs +++ b/crates/node/src/foreign_chain_health.rs @@ -7,18 +7,24 @@ use std::panic::AssertUnwindSafe; use foreign_chain_health_check::{ExpectedIdentities, ProviderResult, Status, check_all_providers}; use futures::FutureExt as _; use mpc_node_config::ForeignChainsConfig; +use tokio::sync::watch; use tracing::{debug, error, info, warn}; use crate::metrics; +/// Healthy provider count per configured chain, keyed by `chain.label()`. +pub type ProviderHealthSnapshot = BTreeMap; + /// Startup healthcheck entrypoint: probes every configured provider against its configured /// expected identity ([`foreign_chain_health_check`]) and logs a per-provider result plus an /// `x/y providers healthy` summary. Publishes per-chain `configured` and `healthy` provider -/// gauges. A probe panic is caught and logged, never propagated (the check is diagnostic-only; -/// the node runs regardless). +/// gauges, plus a healthy-count snapshot over `health_publisher` (surfaced on +/// `/debug/node_config`). A probe panic is caught and logged, never propagated (the check is +/// diagnostic-only; the node runs regardless). pub async fn run_startup_health_check( foreign_chains: ForeignChainsConfig, identities: ExpectedIdentities, + health_publisher: watch::Sender, ) { // Publish `configured` up front, so the gauge is populated even before (or if) the probe runs. for (chain, config) in foreign_chains.iter_chains() { @@ -33,7 +39,7 @@ pub async fn run_startup_health_check( info!("running foreign-chain RPC provider health check"); let results = check_all_providers(&foreign_chains, &identities).await; log_results(&results); - set_healthy_metric(&foreign_chains, &results); + set_healthy_metric(&foreign_chains, &results, &health_publisher); }; if AssertUnwindSafe(probe).catch_unwind().await.is_err() { @@ -44,20 +50,29 @@ pub async fn run_startup_health_check( } /// Sets the per-chain `healthy` gauge to the number of providers that passed, for every -/// configured chain (0 where none passed, so a chain that failed its check reads as `0/N`). -fn set_healthy_metric(foreign_chains: &ForeignChainsConfig, results: &[ProviderResult]) { +/// configured chain (0 where none passed, so a chain that failed its check reads as `0/N`), +/// and pushes the same counts to `health_publisher` for `/debug/node_config`. +fn set_healthy_metric( + foreign_chains: &ForeignChainsConfig, + results: &[ProviderResult], + health_publisher: &watch::Sender, +) { let mut healthy_by_chain: BTreeMap<&str, i64> = BTreeMap::new(); for result in results { if matches!(result.status, Status::Passed) { *healthy_by_chain.entry(result.chain).or_default() += 1; } } + let mut counts = ProviderHealthSnapshot::new(); for (chain, _) in foreign_chains.iter_chains() { let label = chain.label(); + let healthy = healthy_by_chain.get(label).copied().unwrap_or(0); metrics::FOREIGN_CHAIN_RPC_PROVIDERS_HEALTHY .with_label_values(&[label]) - .set(healthy_by_chain.get(label).copied().unwrap_or(0)); + .set(healthy); + counts.insert(label.to_string(), healthy); } + let _ = health_publisher.send(counts); } fn log_results(results: &[ProviderResult]) { @@ -142,18 +157,20 @@ mod tests { base: Some("8453".to_string()), ..Default::default() }; + let (publisher, snapshot) = watch::channel(ProviderHealthSnapshot::new()); // When - run_startup_health_check(foreign_chains, identities).await; + run_startup_health_check(foreign_chains, identities, publisher).await; // Then the probe runs end to end and summarizes the one probed provider (it fails to - // connect, so 0 of 1 are healthy) + // connect, so 0 of 1 are healthy), publishing the healthy count for `base` assert!(logs_contain( "running foreign-chain RPC provider health check" )); assert!(logs_contain( "foreign-chain RPC provider health check complete: 0/1 providers healthy" )); + assert_eq!(snapshot.borrow().get("base"), Some(&0)); } #[test] @@ -241,9 +258,10 @@ mod tests { base: Some(chain_config("http://127.0.0.1:1")), ..Default::default() }; + let (publisher, _snapshot) = watch::channel(ProviderHealthSnapshot::new()); // When - run_startup_health_check(foreign_chains, ExpectedIdentities::default()).await; + run_startup_health_check(foreign_chains, ExpectedIdentities::default(), publisher).await; // Then `configured` is populated up front regardless of the probe outcome assert_eq!( @@ -289,9 +307,15 @@ mod tests { ]; // When - set_healthy_metric(&foreign_chains, &results); + let (publisher, receiver) = watch::channel(ProviderHealthSnapshot::new()); + set_healthy_metric(&foreign_chains, &results, &publisher); - // Then each chain's gauge equals its number of passing providers. + // Then both the gauge and the published snapshot report each chain's passing count, + // including 0 for a chain where nothing passed. + let counts = receiver.borrow(); + assert_eq!(counts.get("base"), Some(&2)); + assert_eq!(counts.get("aptos"), Some(&1)); + assert_eq!(counts.get("ethereum"), Some(&0)); let healthy = |chain| { metrics::FOREIGN_CHAIN_RPC_PROVIDERS_HEALTHY .with_label_values(&[chain]) diff --git a/crates/node/src/run.rs b/crates/node/src/run.rs index 0648403b8..8150426a0 100644 --- a/crates/node/src/run.rs +++ b/crates/node/src/run.rs @@ -184,6 +184,8 @@ pub async fn run_mpc_node(config: StartConfig) -> anyhow::Result<()> { watch::channel(ProtocolContractState::NotInitialized); let (migration_state_sender, migration_state_receiver) = watch::channel((0, BTreeMap::new())); + let (foreign_chains_health_sender, foreign_chains_health_receiver) = + watch::channel(crate::foreign_chain_health::ProviderHealthSnapshot::new()); // Buffer behind the recent-transactions debug page. The indexer forwards // records over `recent_tx_sender`; the drain task records them into the @@ -202,6 +204,7 @@ pub async fn run_mpc_node(config: StartConfig) -> anyhow::Result<()> { protocol_state_receiver, migration_state_receiver, config.node.clone(), + foreign_chains_health_receiver, read_near_config_json(&config.home_dir), recent_transactions.clone(), )) @@ -213,6 +216,7 @@ pub async fn run_mpc_node(config: StartConfig) -> anyhow::Result<()> { root_runtime.spawn(crate::foreign_chain_health::run_startup_health_check( node_config.foreign_chains.clone(), node_config.foreign_chain_health_check.identities.clone(), + foreign_chains_health_sender, )); // Create Indexer and wait for indexer to be synced. diff --git a/crates/node/src/tests.rs b/crates/node/src/tests.rs index c31b049fc..1e2cb33f3 100644 --- a/crates/node/src/tests.rs +++ b/crates/node/src/tests.rs @@ -115,6 +115,8 @@ impl OneNodeTestConfig { let (_, dummy_protocol_state_receiver) = watch::channel(ProtocolContractState::NotInitialized); let (_, dummy_migration_state_receiver) = watch::channel((0, BTreeMap::new())); + let (_, dummy_foreign_chains_health_receiver) = + watch::channel(crate::foreign_chain_health::ProviderHealthSnapshot::new()); // The fake indexer never records, so the buffer stays empty. // TODO(#3522): wire it into the fake indexer and assert on it. let web_server = start_web_server( @@ -125,6 +127,7 @@ impl OneNodeTestConfig { dummy_protocol_state_receiver, dummy_migration_state_receiver, self.config.clone(), + dummy_foreign_chains_health_receiver, serde_json::json!({}), SharedRecentTransactions::default(), ) diff --git a/crates/node/src/web.rs b/crates/node/src/web.rs index 313100143..9fe49a8fd 100644 --- a/crates/node/src/web.rs +++ b/crates/node/src/web.rs @@ -1,4 +1,5 @@ use crate::config::SecretsConfig; +use crate::foreign_chain_health::ProviderHealthSnapshot; use crate::indexer::migrations::ContractMigrationInfo; use crate::tracking::TaskHandle; use axum::body::Body; @@ -68,6 +69,7 @@ struct WebServerState { migration_state_receiver: watch::Receiver<(u64, ContractMigrationInfo)>, static_web_data: StaticWebData, node_config: NodeConfigResponse, + foreign_chains_health: watch::Receiver, nearcore_config: serde_json::Value, /// In-memory log behind the `/debug/recent_transactions` handler. recent_transactions: SharedRecentTransactions, @@ -98,6 +100,9 @@ struct NodeConfigResponse { ckd: CKDConfig, keygen: KeygenConfig, foreign_chains_provider_counts: ForeignChainsProviderCounts, + /// Healthy provider count per configured chain from the startup probe. Empty until the + /// detached probe publishes (mirrors `foreign_chains_provider_counts`, always present). + foreign_chains_provider_health: ProviderHealthSnapshot, cores: Option, separate_asset_generation_runtime: bool, } @@ -118,6 +123,7 @@ impl From for NodeConfigResponse { ckd: config.ckd, keygen: config.keygen, foreign_chains_provider_counts: config.foreign_chains.into(), + foreign_chains_provider_health: ProviderHealthSnapshot::new(), cores: config.cores, separate_asset_generation_runtime: config.separate_asset_generation_runtime, } @@ -183,7 +189,9 @@ async fn debug_tasks(State(state): State) -> String { } async fn debug_node_config(State(state): State) -> Json { - Json(state.node_config.clone()) + let mut response = state.node_config.clone(); + response.foreign_chains_provider_health = state.foreign_chains_health.borrow().clone(); + Json(response) } /// Serves the nearcore `config.json` the embedded indexer runs with. @@ -330,6 +338,7 @@ pub async fn start_web_server( protocol_state_receiver: watch::Receiver, migration_state_receiver: watch::Receiver<(u64, ContractMigrationInfo)>, config: ConfigFile, + foreign_chains_health: watch::Receiver, nearcore_config: serde_json::Value, recent_transactions: SharedRecentTransactions, ) -> anyhow::Result>> { @@ -362,6 +371,7 @@ pub async fn start_web_server( migration_state_receiver, static_web_data, node_config: NodeConfigResponse::from(config), + foreign_chains_health, nearcore_config, recent_transactions, }); @@ -540,11 +550,16 @@ mod tests { #[test] fn node_config_response_json____should_expose_provider_counts_but_no_sensitive_info() { - // Given - let config = test_config(); + // Given a response with the health overlay populated, as `debug_node_config` does at + // request time. + let mut response = NodeConfigResponse::from(test_config()); + response.foreign_chains_provider_health = + [("solana".to_string(), 1i64), ("aptos".to_string(), 0i64)] + .into_iter() + .collect(); // When - let json = serde_json::to_string(&NodeConfigResponse::from(config)).unwrap(); + let json = serde_json::to_string(&response).unwrap(); let value: serde_json::Value = serde_json::from_str(&json).unwrap(); let object = value .as_object() @@ -609,5 +624,30 @@ mod tests { "JSON response must not contain `{needle}`, got: {json}" ); } + + let health = object + .get("foreign_chains_provider_health") + .expect("response must contain `foreign_chains_provider_health`") + .as_object() + .expect("`foreign_chains_provider_health` must be an object"); + assert_eq!(health.get("solana").and_then(|v| v.as_u64()), Some(1)); + assert_eq!(health.get("aptos").and_then(|v| v.as_u64()), Some(0)); + } + + #[test] + fn node_config_response_json____should_present_empty_health_map_before_any_probe() { + // Given a response straight from config, before the probe publishes + let response = NodeConfigResponse::from(test_config()); + + // When + let value: serde_json::Value = + serde_json::from_str(&serde_json::to_string(&response).unwrap()).unwrap(); + + // Then the health field is present as an empty object, never omitted, so it mirrors the + // always-present `foreign_chains_provider_counts`. + assert_eq!( + value["foreign_chains_provider_health"], + serde_json::json!({}) + ); } }