From d0c0f23415789b26f76ca8f2d4e55fc7e83fc550 Mon Sep 17 00:00:00 2001 From: EddieHouston Date: Fri, 21 Aug 2026 16:54:54 +0200 Subject: [PATCH 01/28] fix(rest): keep mempool tx prevout lookup atomic --- src/errors.rs | 9 ++ src/new_index/query.rs | 56 ++++++++++-- src/new_index/schema.rs | 39 +++++---- src/rest.rs | 186 ++++++++++++++++++++++++++-------------- 4 files changed, 199 insertions(+), 91 deletions(-) diff --git a/src/errors.rs b/src/errors.rs index c8d09fc2e..10a1a4d30 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -3,6 +3,10 @@ error_chain! { Error, ErrorKind, ResultExt, Result; } + foreign_links { + RocksDb(rocksdb::Error); + } + errors { Connection(msg: String) { description("Connection error") @@ -34,6 +38,11 @@ error_chain! { display("Too many subscriptions on this connection (limit: {})", limit) } + MissingTxo(outpoint: String) { + description("Missing transaction output") + display("missing txo {}", outpoint) + } + InvalidParams(msg: String) { description("Invalid RPC params") display("{}", msg) diff --git a/src/new_index/query.rs b/src/new_index/query.rs index 14a851f5c..c8fe80a6f 100644 --- a/src/new_index/query.rs +++ b/src/new_index/query.rs @@ -77,6 +77,10 @@ impl Query { self.mempool.read().unwrap() } + pub fn with_mempool(&self, f: impl FnOnce(&Mempool) -> T) -> T { + with_read_snapshot(&self.mempool, f) + } + #[trace] pub fn broadcast_raw(&self, txhex: &str) -> Result { let txid = self.daemon.broadcast_raw(txhex)?; @@ -152,10 +156,11 @@ impl Query { } #[trace] - pub fn lookup_txn(&self, txid: &Txid) -> Option { - self.chain - .lookup_txn(txid, None) - .or_else(|| self.mempool().lookup_txn(txid)) + pub fn lookup_txn(&self, txid: &Txid) -> Result> { + Ok(self + .chain + .lookup_txn(txid, None)? + .or_else(|| self.mempool().lookup_txn(txid))) } #[trace] @@ -166,11 +171,9 @@ impl Query { } #[trace] - pub fn lookup_txos(&self, outpoints: BTreeSet) -> HashMap { + pub fn lookup_txos(&self, outpoints: BTreeSet) -> Result> { // the mempool lookup_txos() internally looks up confirmed txos as well - self.mempool() - .lookup_txos(outpoints) - .expect("failed loading txos") + self.mempool().lookup_txos(outpoints) } #[trace] @@ -356,3 +359,40 @@ impl Query { Ok((total_num, results)) } } + +fn with_read_snapshot(lock: &RwLock, f: impl FnOnce(&T) -> R) -> R { + let guard = lock.read().unwrap(); + f(&guard) +} + +#[cfg(test)] +mod tests { + use super::with_read_snapshot; + use std::sync::{mpsc, Arc, RwLock}; + use std::thread; + + #[test] + fn read_snapshot_blocks_writes_between_selection_and_lookup() { + let state = Arc::new(RwLock::new(vec!["parent", "child"])); + let writer_state = Arc::clone(&state); + let (selected_tx, selected_rx) = mpsc::channel(); + let (continue_tx, continue_rx) = mpsc::channel(); + + let reader = thread::spawn(move || { + with_read_snapshot(&state, |snapshot| { + assert_eq!(snapshot.last(), Some(&"child")); + selected_tx.send(()).unwrap(); + continue_rx.recv().unwrap(); + assert_eq!(snapshot.first(), Some(&"parent")); + }); + }); + + selected_rx.recv().unwrap(); + assert!(writer_state.try_write().is_err()); + continue_tx.send(()).unwrap(); + reader.join().unwrap(); + + writer_state.write().unwrap().remove(0); + assert_eq!(*writer_state.read().unwrap(), vec!["child"]); + } +} diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index 2ef6ce74d..fead9decb 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -777,7 +777,7 @@ impl ChainQuery { scripthash: &[u8], last_seen_txid: Option<&Txid>, limit: usize, - ) -> Vec<(Transaction, BlockId)> { + ) -> Result> { // scripthash lookup self._history(b'H', scripthash, last_seen_txid, limit) } @@ -788,7 +788,7 @@ impl ChainQuery { hash: &[u8], last_seen_txid: Option<&Txid>, limit: usize, - ) -> Vec<(Transaction, BlockId)> { + ) -> Result> { let _timer_scan = self.start_timer("history"); let headers = self.store.indexed_headers.read().unwrap(); let history_iter = self @@ -818,12 +818,12 @@ impl ChainQuery { } drop(headers); - self.lookup_txns(&txids_with_blockhash) - .expect("failed looking up txs in history index") + Ok(self + .lookup_txns(&txids_with_blockhash)? .into_iter() .zip(blockids) .map(|(tx, blockid)| (tx, blockid)) - .collect() + .collect()) } pub fn history_txids(&self, scripthash: &[u8], limit: usize) -> Vec<(Txid, BlockId)> { @@ -1137,17 +1137,21 @@ impl ChainQuery { pub fn lookup_txns(&self, txids: &[(Txid, BlockHash)]) -> Result> { let _timer = self.start_timer("lookup_txns"); - Ok(self - .lookup_raw_txns(txids)? + self.lookup_raw_txns(txids)? .into_iter() - .map(|rawtx| deserialize(&rawtx).expect("failed to parse Transaction")) - .collect()) + .map(|rawtx| deserialize(&rawtx).chain_err(|| "failed to parse Transaction")) + .collect() } - pub fn lookup_txn(&self, txid: &Txid, blockhash: Option<&BlockHash>) -> Option { + pub fn lookup_txn( + &self, + txid: &Txid, + blockhash: Option<&BlockHash>, + ) -> Result> { let _timer = self.start_timer("lookup_txn"); - let rawtx = self.lookup_raw_txn(txid, blockhash)?; - Some(deserialize(&rawtx).expect("failed to parse Transaction")) + self.lookup_raw_txn(txid, blockhash) + .map(|rawtx| deserialize(&rawtx).chain_err(|| "failed to parse Transaction")) + .transpose() } pub fn lookup_raw_txns(&self, txids: &[(Txid, BlockHash)]) -> Result> { @@ -1166,7 +1170,7 @@ impl ChainQuery { .txstore_db .multi_get(keys) .into_iter() - .map(|val| val.unwrap().chain_err(|| "missing tx")) + .map(|val| val?.chain_err(|| "missing tx")) .collect() } } @@ -1292,7 +1296,7 @@ impl ChainQuery { asset_id: &AssetId, last_seen_txid: Option<&Txid>, limit: usize, - ) -> Vec<(Transaction, BlockId)> { + ) -> Result> { self._history(b'I', &asset_id.into_inner()[..], last_seen_txid, limit) } @@ -1390,10 +1394,9 @@ fn lookup_txos(txstore_db: &DB, outpoints: BTreeSet) -> Result, query: &Query) -> u32 { }) } -fn prepare_txs( +fn prepare_txs( txs: Vec<(Transaction, Option)>, - query: &Query, + lookup_txos: F, config: &Config, -) -> Vec { +) -> Result, HttpError> +where + F: FnOnce(BTreeSet) -> errors::Result>, +{ let outpoints = txs .iter() .flat_map(|(tx, _)| { @@ -502,11 +505,12 @@ fn prepare_txs( }) .collect(); - let prevouts = query.lookup_txos(outpoints); + let prevouts = lookup_txos(outpoints).map_err(HttpError::lookup)?; - txs.into_iter() + Ok(txs + .into_iter() .map(|(tx, blockid)| TransactionValue::new(tx, blockid, &prevouts, config)) - .collect() + .collect()) } fn spawn_conn( @@ -849,7 +853,7 @@ fn handle_blocking_request( } (&Method::GET, Some(&"block"), Some(hash), Some(&"txs"), start_index, None) => { let hash = BlockHash::from_str(hash)?; - + // Add lightweight validation that block exists before fetching transactions, // to avoid expensive lookups in case of invalid block hash query.chain().get_block_header(&hash) @@ -878,7 +882,10 @@ fn handle_blocking_request( // XXX stale blocks alway get TTL_SHORT let ttl = ttl_by_depth(blockid.map(|b| b.height), query); - json_response(prepare_txs(txs, query, config), ttl) + json_response( + prepare_txs(txs, |outpoints| query.chain().lookup_txos(outpoints), config)?, + ttl, + ) } (&Method::GET, Some(script_type @ &"address"), Some(script_str), None, None, None) | (&Method::GET, Some(script_type @ &"scripthash"), Some(script_str), None, None, None) => { @@ -910,26 +917,26 @@ fn handle_blocking_request( None, ) => { let script_hash = to_scripthash(script_type, script_str, config.network_type)?; + let chain_txs = query + .chain() + .history(&script_hash[..], None, CHAIN_TXS_PER_PAGE) + .map_err(HttpError::lookup)?; - let mut txs = vec![]; - - txs.extend( - query - .mempool() + let txs = query.with_mempool(|mempool| { + let mut txs = mempool .history(&script_hash[..], MAX_MEMPOOL_TXS) .into_iter() - .map(|tx| (tx, None)), - ); - - txs.extend( - query - .chain() - .history(&script_hash[..], None, CHAIN_TXS_PER_PAGE) - .into_iter() - .map(|(tx, blockid)| (tx, Some(blockid))), - ); + .map(|tx| (tx, None)) + .collect::>(); + txs.extend( + chain_txs + .into_iter() + .map(|(tx, blockid)| (tx, Some(blockid))), + ); + prepare_txs(txs, |outpoints| mempool.lookup_txos(outpoints), config) + })?; - json_response(prepare_txs(txs, query, config), TTL_SHORT) + json_response(txs, TTL_SHORT) } ( @@ -950,7 +957,6 @@ fn handle_blocking_request( ) => { let script_hash = to_scripthash(script_type, script_str, config.network_type)?; let last_seen_txid = last_seen_txid.and_then(|txid| Txid::from_str(txid).ok()); - let txs = query .chain() .history( @@ -958,11 +964,15 @@ fn handle_blocking_request( last_seen_txid.as_ref(), CHAIN_TXS_PER_PAGE, ) + .map_err(HttpError::lookup)? .into_iter() .map(|(tx, blockid)| (tx, Some(blockid))) .collect(); - json_response(prepare_txs(txs, query, config), TTL_SHORT) + json_response( + prepare_txs(txs, |outpoints| query.chain().lookup_txos(outpoints), config)?, + TTL_SHORT, + ) } ( &Method::GET, @@ -981,15 +991,16 @@ fn handle_blocking_request( None, ) => { let script_hash = to_scripthash(script_type, script_str, config.network_type)?; + let txs = query.with_mempool(|mempool| { + let txs = mempool + .history(&script_hash[..], MAX_MEMPOOL_TXS) + .into_iter() + .map(|tx| (tx, None)) + .collect(); + prepare_txs(txs, |outpoints| mempool.lookup_txos(outpoints), config) + })?; - let txs = query - .mempool() - .history(&script_hash[..], MAX_MEMPOOL_TXS) - .into_iter() - .map(|tx| (tx, None)) - .collect(); - - json_response(prepare_txs(txs, query, config), TTL_SHORT) + json_response(txs, TTL_SHORT) } ( @@ -1026,13 +1037,34 @@ fn handle_blocking_request( } (&Method::GET, Some(&"tx"), Some(hash), None, None, None) => { let hash = Txid::from_str(hash)?; - let tx = query - .lookup_txn(&hash) - .ok_or_else(|| HttpError::not_found("Transaction not found".to_string()))?; + let chain_tx = query + .chain() + .lookup_txn(&hash, None) + .map_err(HttpError::lookup)?; let blockid = query.chain().tx_confirming_block(&hash); let ttl = ttl_by_depth(blockid.as_ref().map(|b| b.height), query); - let tx = prepare_txs(vec![(tx, blockid)], query, config).remove(0); + let tx = match chain_tx { + Some(tx) => prepare_txs( + vec![(tx, blockid)], + |outpoints| query.chain().lookup_txos(outpoints), + config, + )? + .remove(0), + None => query.with_mempool(|mempool| { + let tx = mempool + .lookup_txn(&hash) + .ok_or_else(|| HttpError::not_found("Transaction not found".to_string()))?; + Ok::<_, HttpError>( + prepare_txs( + vec![(tx, None)], + |outpoints| mempool.lookup_txos(outpoints), + config, + )? + .remove(0), + ) + })?, + }; json_response(tx, ttl) } @@ -1118,6 +1150,7 @@ fn handle_blocking_request( let hash = Txid::from_str(hash)?; let tx = query .lookup_txn(&hash) + .map_err(HttpError::lookup)? .ok_or_else(|| HttpError::not_found("Transaction not found".to_string()))?; let spends: Vec = query .lookup_tx_spends(&tx) @@ -1257,26 +1290,26 @@ fn handle_blocking_request( #[cfg(feature = "liquid")] (&Method::GET, Some(&"asset"), Some(asset_str), Some(&"txs"), None, None) => { let asset_id = AssetId::from_str(asset_str)?; + let chain_txs = query + .chain() + .asset_history(&asset_id, None, CHAIN_TXS_PER_PAGE) + .map_err(HttpError::lookup)?; - let mut txs = vec![]; - - txs.extend( - query - .mempool() + let txs = query.with_mempool(|mempool| { + let mut txs = mempool .asset_history(&asset_id, MAX_MEMPOOL_TXS) .into_iter() - .map(|tx| (tx, None)), - ); - - txs.extend( - query - .chain() - .asset_history(&asset_id, None, CHAIN_TXS_PER_PAGE) - .into_iter() - .map(|(tx, blockid)| (tx, Some(blockid))), - ); + .map(|tx| (tx, None)) + .collect::>(); + txs.extend( + chain_txs + .into_iter() + .map(|(tx, blockid)| (tx, Some(blockid))), + ); + prepare_txs(txs, |outpoints| mempool.lookup_txos(outpoints), config) + })?; - json_response(prepare_txs(txs, query, config), TTL_SHORT) + json_response(txs, TTL_SHORT) } #[cfg(feature = "liquid")] @@ -1290,29 +1323,33 @@ fn handle_blocking_request( ) => { let asset_id = AssetId::from_str(asset_str)?; let last_seen_txid = last_seen_txid.and_then(|txid| Txid::from_str(txid).ok()); - let txs = query .chain() .asset_history(&asset_id, last_seen_txid.as_ref(), CHAIN_TXS_PER_PAGE) + .map_err(HttpError::lookup)? .into_iter() .map(|(tx, blockid)| (tx, Some(blockid))) .collect(); - json_response(prepare_txs(txs, query, config), TTL_SHORT) + json_response( + prepare_txs(txs, |outpoints| query.chain().lookup_txos(outpoints), config)?, + TTL_SHORT, + ) } #[cfg(feature = "liquid")] (&Method::GET, Some(&"asset"), Some(asset_str), Some(&"txs"), Some(&"mempool"), None) => { let asset_id = AssetId::from_str(asset_str)?; + let txs = query.with_mempool(|mempool| { + let txs = mempool + .asset_history(&asset_id, MAX_MEMPOOL_TXS) + .into_iter() + .map(|tx| (tx, None)) + .collect(); + prepare_txs(txs, |outpoints| mempool.lookup_txos(outpoints), config) + })?; - let txs = query - .mempool() - .asset_history(&asset_id, MAX_MEMPOOL_TXS) - .into_iter() - .map(|tx| (tx, None)) - .collect(); - - json_response(prepare_txs(txs, query, config), TTL_SHORT) + json_response(txs, TTL_SHORT) } #[cfg(feature = "liquid")] @@ -1527,6 +1564,14 @@ impl HttpError { fn forbidden(msg: String) -> Self { HttpError(StatusCode::FORBIDDEN, msg) } + + fn lookup(err: errors::Error) -> Self { + let status = match err.kind() { + errors::ErrorKind::MissingTxo(_) => StatusCode::NOT_FOUND, + _ => StatusCode::SERVICE_UNAVAILABLE, + }; + HttpError(status, err.to_string()) + } } impl From for HttpError { @@ -1677,6 +1722,17 @@ mod tests { assert_eq!(rejected.0, StatusCode::BAD_REQUEST); } + #[test] + fn transaction_preparation_failures_map_to_not_found_or_unavailable() { + let missing = HttpError::lookup(errors::Error::from(ErrorKind::MissingTxo( + "abc:0".to_string(), + ))); + assert_eq!(missing.0, StatusCode::NOT_FOUND); + + let corrupt = HttpError::lookup(errors::Error::from("failed to parse TxOut")); + assert_eq!(corrupt.0, StatusCode::SERVICE_UNAVAILABLE); + } + #[test] fn test_parse_query_param() { let mut query_params = HashMap::new(); From 42097953e8a169d4387e8504a66e5cd21e8c99c4 Mon Sep 17 00:00:00 2001 From: Philippe McLean Date: Fri, 21 Aug 2026 10:42:33 -0700 Subject: [PATCH 02/28] add test that demonstrates abort under mempool eviction events --- tests/mempool_eviction_race.rs | 159 +++++++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 tests/mempool_eviction_race.rs diff --git a/tests/mempool_eviction_race.rs b/tests/mempool_eviction_race.rs new file mode 100644 index 000000000..d99431137 --- /dev/null +++ b/tests/mempool_eviction_race.rs @@ -0,0 +1,159 @@ +//! Deterministic reproduction of the mempool-eviction prevout-lookup abort +//! this branch fixes (see src/new_index/query.rs's `Query::lookup_txos`). +//! +//! The REST handlers for e.g. GET /tx/:txid used to look up a transaction +//! (once, in one mempool snapshot) and then separately resolve its prevouts +//! via `Query::lookup_txos()` (a second, later mempool snapshot). If the +//! referenced mempool ancestor was evicted in between -- e.g. by an RBF +//! replacement -- the second lookup could come back empty and hit an +//! `.expect("failed loading txos")`. +//! +//! Rather than racing real wall-clock timing against electrs's background +//! mempool-sync thread (which only runs on a fixed poll interval and makes +//! the window practically unhittable black-box), this test drives the same +//! two steps directly and deterministically: select the transaction, then +//! explicitly evict its ancestor by replacing it on the node and calling +//! `Mempool::update()` inline (this is exactly what the background sync +//! loop does, just invoked synchronously instead of on a timer), then +//! resolve prevouts. No timing luck involved. + +use std::collections::BTreeSet; + +use serde_json::json; + +use electrs::chain::OutPoint; +use electrs::errors::ErrorKind; +use electrs::new_index::Mempool; + +pub mod common; + +use common::Result; + +#[test] +fn prevout_lookup_survives_ancestor_eviction() -> Result<()> { + let mut tester = common::TestRunner::new()?; + + // Parent: a wallet spend, left unconfirmed. + let addr_parent = tester.newaddress()?; + let parent_txid = tester.send(&addr_parent, "1.0 BTC".parse().unwrap())?; + + let parent_tx = tester.get_raw_transaction(parent_txid)?; + let parent_spk = addr_parent.script_pubkey(); + let (parent_vout, parent_txout) = parent_tx + .output + .iter() + .enumerate() + .find(|(_, o)| o.script_pubkey == parent_spk) + .expect("funding output not found in parent tx"); + + // Child: explicitly spends the parent's own unconfirmed output, so the + // ancestor relationship (and thus the prevout dependency) is guaranteed. + let addr_child = tester.newaddress()?; + let child_value = parent_txout.value - bitcoin::Amount::from_sat(1000); + let raw_child: String = tester.node_client().call( + "createrawtransaction", + &[ + json!([{ "txid": parent_txid.to_string(), "vout": parent_vout }]), + json!({ addr_child.to_string(): child_value.to_btc() }), + ], + )?; + let signed: serde_json::Value = tester + .node_client() + .call("signrawtransactionwithwallet", &[json!(raw_child)])?; + assert_eq!(signed["complete"].as_bool(), Some(true)); + let child_txid_str: String = tester + .node_client() + .call("sendrawtransaction", &[signed["hex"].clone()])?; + let child_txid = child_txid_str.parse().unwrap(); + + tester.sync()?; + + let query = tester.query(); + + // SELECT: mirrors the GET /tx/:txid handler's `query.lookup_txn()` -- + // one mempool snapshot, cloning out the child tx. + let child_tx = query + .lookup_txn(&child_txid)? + .expect("child tx should be indexed before eviction"); + let outpoints: BTreeSet = child_tx + .input + .iter() + .map(|txin| txin.previous_output) + .collect(); + assert!(outpoints.contains(&OutPoint::new(parent_txid, parent_vout as u32))); + + // Sanity check: resolving prevouts works fine right now, before any + // eviction -- proves the graceful error below is caused by the + // eviction, not by some unrelated test-setup mistake. + let prevouts_before = query.lookup_txos(outpoints.clone())?; + assert!(prevouts_before.contains_key(&OutPoint::new(parent_txid, parent_vout as u32))); + + // EVICT: RBF-replace the parent on the node by directly double-spending + // its inputs (not via `bumpfee`, which refuses once the wallet sees a + // real descendant -- exactly the descendant we just created). This also + // invalidates and evicts the child, since its input no longer exists. + // Then apply that eviction to electrs's local mempool view -- exactly + // what the background sync loop does on its next tick, just called + // directly and synchronously here instead of racing its timer. + let gettx: serde_json::Value = tester + .node_client() + .call("gettransaction", &[json!(parent_txid.to_string())])?; + let parent_fee = bitcoin::Amount::from_btc(gettx["fee"].as_f64().unwrap().abs()).unwrap(); + let total_output: bitcoin::Amount = parent_tx.output.iter().map(|o| o.value).sum(); + let total_input = total_output + parent_fee; + let higher_fee = bitcoin::Amount::from_sat(50_000); // well above the parent+child's combined fee + let replacement_value = total_input - higher_fee; + + let addr_conflict = tester.newaddress()?; + let raw_conflict: String = tester.node_client().call( + "createrawtransaction", + &[ + json!(parent_tx + .input + .iter() + .map(|txin| json!({ + "txid": txin.previous_output.txid.to_string(), + "vout": txin.previous_output.vout, + })) + .collect::>()), + json!({ addr_conflict.to_string(): replacement_value.to_btc() }), + ], + )?; + let signed_conflict: serde_json::Value = tester + .node_client() + .call("signrawtransactionwithwallet", &[json!(raw_conflict)])?; + assert_eq!(signed_conflict["complete"].as_bool(), Some(true)); + let _replacement_txid: String = tester + .node_client() + .call("sendrawtransaction", &[signed_conflict["hex"].clone()])?; + + let tip = tester.get_best_block_hash()?; + assert!(Mempool::update(&tester.mempool(), &tester.daemon(), &tip)?); + + // LOOKUP: mirrors prepare_txs()'s `query.lookup_txos()` on the *same* + // outpoints selected before the eviction -- the parent's output is now + // gone from both the local mempool view (evicted) and the confirmed + // chain (it was never mined). Pre-fix this panicked; post-fix it must + // come back as a graceful `MissingTxo` error instead. + let result = query.lookup_txos(outpoints); + + match result { + Ok(txos) => panic!( + "expected lookup_txos() to report the evicted prevout as missing, \ + but it returned successfully with {} txos -- the race wasn't \ + reproduced (check the eviction actually happened)", + txos.len() + ), + Err(ref e) => match e.kind() { + ErrorKind::MissingTxo(outpoint) => { + assert_eq!( + outpoint, + &OutPoint::new(parent_txid, parent_vout as u32).to_string() + ); + } + other => panic!("expected ErrorKind::MissingTxo, got: {:?}", other), + }, + } + + Ok(()) +} From 03ed55cd647c184b07bbe45611bf239ca035cc6a Mon Sep 17 00:00:00 2001 From: EddieHouston Date: Mon, 24 Aug 2026 11:49:19 +0200 Subject: [PATCH 03/28] fix(rest): address review feedback on mempool snapshot atomicity - Query::mempool() recovers from a poisoned lock via unwrap_or_else(|e| e.into_inner()) rather than unwrapping. Under panic=abort the process terminates before a poison marker is observable, but serving stale reads is strictly better than aborting the front-end for anyone who turns that off later. - Remove with_mempool() and with_read_snapshot() along with the test that only covered the helper. The five REST call sites now bind the guard directly via let mempool = query.mempool(); with an explicit drop(mempool); at the end of the atomic section. The atomicity guarantee comes from scope, and the codebase keeps a single mempool-access idiom. - Delete Query::lookup_txos. With prepare_txs taking a lookup closure instead of a HashMap, the wrapper had no non-test callers. - HttpError::lookup emits a warn log before returning so operators can see what is driving REST 404 and 503 responses. - Note the pre-existing race between chain.history() and the mempool snapshot in the address and asset history handlers. - Adjust tests/mempool_eviction_race.rs to call query.mempool().lookup_txos(...) directly instead of the deleted Query::lookup_txos wrapper. Two separate mempool snapshots is exactly the pre-fix code path the test needs to exercise. Also add query, mempool, and daemon accessors on TestRunner in tests/common.rs. --- src/new_index/query.rs | 50 +-------------- src/rest.rs | 108 ++++++++++++++++++--------------- tests/common.rs | 12 ++++ tests/mempool_eviction_race.rs | 32 ++++++---- 4 files changed, 91 insertions(+), 111 deletions(-) diff --git a/src/new_index/query.rs b/src/new_index/query.rs index c8fe80a6f..12c4f4d49 100644 --- a/src/new_index/query.rs +++ b/src/new_index/query.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeSet, HashMap}; use std::sync::{Arc, Mutex, RwLock, RwLockReadGuard, TryLockError}; use std::time::{Duration, Instant}; -use crate::chain::{Network, OutPoint, Transaction, TxOut, Txid}; +use crate::chain::{Network, OutPoint, Transaction, Txid}; use crate::config::Config; use crate::daemon::{Daemon, SubmitPackageResult}; use crate::errors::*; @@ -74,11 +74,7 @@ impl Query { } pub fn mempool(&self) -> RwLockReadGuard<'_, Mempool> { - self.mempool.read().unwrap() - } - - pub fn with_mempool(&self, f: impl FnOnce(&Mempool) -> T) -> T { - with_read_snapshot(&self.mempool, f) + self.mempool.read().unwrap_or_else(|e| e.into_inner()) } #[trace] @@ -170,12 +166,6 @@ impl Query { .or_else(|| self.mempool().lookup_raw_txn(txid)) } - #[trace] - pub fn lookup_txos(&self, outpoints: BTreeSet) -> Result> { - // the mempool lookup_txos() internally looks up confirmed txos as well - self.mempool().lookup_txos(outpoints) - } - #[trace] pub fn lookup_spend(&self, outpoint: &OutPoint) -> Option { self.chain @@ -360,39 +350,3 @@ impl Query { } } -fn with_read_snapshot(lock: &RwLock, f: impl FnOnce(&T) -> R) -> R { - let guard = lock.read().unwrap(); - f(&guard) -} - -#[cfg(test)] -mod tests { - use super::with_read_snapshot; - use std::sync::{mpsc, Arc, RwLock}; - use std::thread; - - #[test] - fn read_snapshot_blocks_writes_between_selection_and_lookup() { - let state = Arc::new(RwLock::new(vec!["parent", "child"])); - let writer_state = Arc::clone(&state); - let (selected_tx, selected_rx) = mpsc::channel(); - let (continue_tx, continue_rx) = mpsc::channel(); - - let reader = thread::spawn(move || { - with_read_snapshot(&state, |snapshot| { - assert_eq!(snapshot.last(), Some(&"child")); - selected_tx.send(()).unwrap(); - continue_rx.recv().unwrap(); - assert_eq!(snapshot.first(), Some(&"parent")); - }); - }); - - selected_rx.recv().unwrap(); - assert!(writer_state.try_write().is_err()); - continue_tx.send(()).unwrap(); - reader.join().unwrap(); - - writer_state.write().unwrap().remove(0); - assert_eq!(*writer_state.read().unwrap(), vec!["child"]); - } -} diff --git a/src/rest.rs b/src/rest.rs index 9c8d827cc..1775b635d 100644 --- a/src/rest.rs +++ b/src/rest.rs @@ -917,24 +917,27 @@ fn handle_blocking_request( None, ) => { let script_hash = to_scripthash(script_type, script_str, config.network_type)?; + // Read outside the mempool guard, so a tx that is both indexed as confirmed + // and still in the local mempool can appear in chain_txs and the mempool + // history below. let chain_txs = query .chain() .history(&script_hash[..], None, CHAIN_TXS_PER_PAGE) .map_err(HttpError::lookup)?; - let txs = query.with_mempool(|mempool| { - let mut txs = mempool - .history(&script_hash[..], MAX_MEMPOOL_TXS) + let mempool = query.mempool(); + let mut txs = mempool + .history(&script_hash[..], MAX_MEMPOOL_TXS) + .into_iter() + .map(|tx| (tx, None)) + .collect::>(); + txs.extend( + chain_txs .into_iter() - .map(|tx| (tx, None)) - .collect::>(); - txs.extend( - chain_txs - .into_iter() - .map(|(tx, blockid)| (tx, Some(blockid))), - ); - prepare_txs(txs, |outpoints| mempool.lookup_txos(outpoints), config) - })?; + .map(|(tx, blockid)| (tx, Some(blockid))), + ); + let txs = prepare_txs(txs, |outpoints| mempool.lookup_txos(outpoints), config)?; + drop(mempool); json_response(txs, TTL_SHORT) } @@ -991,14 +994,14 @@ fn handle_blocking_request( None, ) => { let script_hash = to_scripthash(script_type, script_str, config.network_type)?; - let txs = query.with_mempool(|mempool| { - let txs = mempool - .history(&script_hash[..], MAX_MEMPOOL_TXS) - .into_iter() - .map(|tx| (tx, None)) - .collect(); - prepare_txs(txs, |outpoints| mempool.lookup_txos(outpoints), config) - })?; + let mempool = query.mempool(); + let txs = mempool + .history(&script_hash[..], MAX_MEMPOOL_TXS) + .into_iter() + .map(|tx| (tx, None)) + .collect(); + let txs = prepare_txs(txs, |outpoints| mempool.lookup_txos(outpoints), config)?; + drop(mempool); json_response(txs, TTL_SHORT) } @@ -1051,19 +1054,20 @@ fn handle_blocking_request( config, )? .remove(0), - None => query.with_mempool(|mempool| { + None => { + let mempool = query.mempool(); let tx = mempool .lookup_txn(&hash) .ok_or_else(|| HttpError::not_found("Transaction not found".to_string()))?; - Ok::<_, HttpError>( - prepare_txs( - vec![(tx, None)], - |outpoints| mempool.lookup_txos(outpoints), - config, - )? - .remove(0), - ) - })?, + let prepared = prepare_txs( + vec![(tx, None)], + |outpoints| mempool.lookup_txos(outpoints), + config, + )? + .remove(0); + drop(mempool); + prepared + } }; json_response(tx, ttl) @@ -1290,24 +1294,27 @@ fn handle_blocking_request( #[cfg(feature = "liquid")] (&Method::GET, Some(&"asset"), Some(asset_str), Some(&"txs"), None, None) => { let asset_id = AssetId::from_str(asset_str)?; + // Read outside the mempool guard, so a tx that is both indexed as confirmed + // and still in the local mempool can appear in chain_txs and the mempool + // history below. let chain_txs = query .chain() .asset_history(&asset_id, None, CHAIN_TXS_PER_PAGE) .map_err(HttpError::lookup)?; - let txs = query.with_mempool(|mempool| { - let mut txs = mempool - .asset_history(&asset_id, MAX_MEMPOOL_TXS) + let mempool = query.mempool(); + let mut txs = mempool + .asset_history(&asset_id, MAX_MEMPOOL_TXS) + .into_iter() + .map(|tx| (tx, None)) + .collect::>(); + txs.extend( + chain_txs .into_iter() - .map(|tx| (tx, None)) - .collect::>(); - txs.extend( - chain_txs - .into_iter() - .map(|(tx, blockid)| (tx, Some(blockid))), - ); - prepare_txs(txs, |outpoints| mempool.lookup_txos(outpoints), config) - })?; + .map(|(tx, blockid)| (tx, Some(blockid))), + ); + let txs = prepare_txs(txs, |outpoints| mempool.lookup_txos(outpoints), config)?; + drop(mempool); json_response(txs, TTL_SHORT) } @@ -1340,14 +1347,14 @@ fn handle_blocking_request( #[cfg(feature = "liquid")] (&Method::GET, Some(&"asset"), Some(asset_str), Some(&"txs"), Some(&"mempool"), None) => { let asset_id = AssetId::from_str(asset_str)?; - let txs = query.with_mempool(|mempool| { - let txs = mempool - .asset_history(&asset_id, MAX_MEMPOOL_TXS) - .into_iter() - .map(|tx| (tx, None)) - .collect(); - prepare_txs(txs, |outpoints| mempool.lookup_txos(outpoints), config) - })?; + let mempool = query.mempool(); + let txs = mempool + .asset_history(&asset_id, MAX_MEMPOOL_TXS) + .into_iter() + .map(|tx| (tx, None)) + .collect(); + let txs = prepare_txs(txs, |outpoints| mempool.lookup_txos(outpoints), config)?; + drop(mempool); json_response(txs, TTL_SHORT) } @@ -1570,6 +1577,7 @@ impl HttpError { errors::ErrorKind::MissingTxo(_) => StatusCode::NOT_FOUND, _ => StatusCode::SERVICE_UNAVAILABLE, }; + warn!("REST lookup failed status='{}' err='{:?}'", status, err); HttpError(status, err.to_string()) } } diff --git a/tests/common.rs b/tests/common.rs index 7bcd19d82..c3f479050 100644 --- a/tests/common.rs +++ b/tests/common.rs @@ -227,6 +227,18 @@ impl TestRunner { return &self.node.client(); } + pub fn query(&self) -> &Arc { + &self.query + } + + pub fn mempool(&self) -> &Arc> { + &self.mempool + } + + pub fn daemon(&self) -> &Arc { + &self.daemon + } + pub fn sync(&mut self) -> Result<()> { let tip = self.indexer.update(&self.daemon)?; assert!(Mempool::update(&self.mempool, &self.daemon, &tip)?); diff --git a/tests/mempool_eviction_race.rs b/tests/mempool_eviction_race.rs index d99431137..34d7fb90f 100644 --- a/tests/mempool_eviction_race.rs +++ b/tests/mempool_eviction_race.rs @@ -1,12 +1,17 @@ //! Deterministic reproduction of the mempool-eviction prevout-lookup abort -//! this branch fixes (see src/new_index/query.rs's `Query::lookup_txos`). +//! in src/new_index/query.rs and src/rest.rs. //! //! The REST handlers for e.g. GET /tx/:txid used to look up a transaction //! (once, in one mempool snapshot) and then separately resolve its prevouts -//! via `Query::lookup_txos()` (a second, later mempool snapshot). If the -//! referenced mempool ancestor was evicted in between -- e.g. by an RBF -//! replacement -- the second lookup could come back empty and hit an -//! `.expect("failed loading txos")`. +//! via a second, later mempool snapshot. If the referenced mempool ancestor +//! was evicted in between -- e.g. by an RBF replacement -- the second lookup +//! could come back empty and hit an `.expect("failed loading txos")`. +//! +//! This test drives those two separate snapshots directly: SELECT via +//! `Query::lookup_txn()`, then LOOKUP via `Query::mempool().lookup_txos()`. +//! The fix in this MR removes the panic by propagating the miss as a +//! `MissingTxo` error and by holding a single snapshot across both steps in +//! the REST handlers themselves. //! //! Rather than racing real wall-clock timing against electrs's background //! mempool-sync thread (which only runs on a fixed poll interval and makes @@ -85,7 +90,7 @@ fn prevout_lookup_survives_ancestor_eviction() -> Result<()> { // Sanity check: resolving prevouts works fine right now, before any // eviction -- proves the graceful error below is caused by the // eviction, not by some unrelated test-setup mistake. - let prevouts_before = query.lookup_txos(outpoints.clone())?; + let prevouts_before = query.mempool().lookup_txos(outpoints.clone())?; assert!(prevouts_before.contains_key(&OutPoint::new(parent_txid, parent_vout as u32))); // EVICT: RBF-replace the parent on the node by directly double-spending @@ -130,12 +135,13 @@ fn prevout_lookup_survives_ancestor_eviction() -> Result<()> { let tip = tester.get_best_block_hash()?; assert!(Mempool::update(&tester.mempool(), &tester.daemon(), &tip)?); - // LOOKUP: mirrors prepare_txs()'s `query.lookup_txos()` on the *same* - // outpoints selected before the eviction -- the parent's output is now - // gone from both the local mempool view (evicted) and the confirmed - // chain (it was never mined). Pre-fix this panicked; post-fix it must - // come back as a graceful `MissingTxo` error instead. - let result = query.lookup_txos(outpoints); + // LOOKUP: a second, independent mempool snapshot on the *same* outpoints + // selected before the eviction -- the parent's output is now gone from + // both the local mempool view (evicted) and the confirmed chain (it was + // never mined). Pre-fix this panicked at the `.expect("failed loading + // txos")` in the `Query::lookup_txos` wrapper; post-fix it must come + // back as a graceful `MissingTxo` error instead. + let result = query.mempool().lookup_txos(outpoints); match result { Ok(txos) => panic!( @@ -148,7 +154,7 @@ fn prevout_lookup_survives_ancestor_eviction() -> Result<()> { ErrorKind::MissingTxo(outpoint) => { assert_eq!( outpoint, - &OutPoint::new(parent_txid, parent_vout as u32).to_string() + &OutPoint::new(parent_txid, parent_vout as u32).to_string()[..] ); } other => panic!("expected ErrorKind::MissingTxo, got: {:?}", other), From 8a8ac15805ea78c2054219761db84bd2732a0503 Mon Sep 17 00:00:00 2001 From: EddieHouston Date: Mon, 24 Aug 2026 14:23:49 +0200 Subject: [PATCH 04/28] test(rest): verify mempool snapshot blocks eviction Keep one mempool read guard across transaction selection and prevout resolution in the eviction regression test. Assert that a writer cannot acquire the lock during this atomic section and can acquire it after the snapshot is released. Retain coverage that an evicted prevout returns MissingTxo instead of panicking. --- tests/mempool_eviction_race.rs | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/tests/mempool_eviction_race.rs b/tests/mempool_eviction_race.rs index 34d7fb90f..79d0c29aa 100644 --- a/tests/mempool_eviction_race.rs +++ b/tests/mempool_eviction_race.rs @@ -7,11 +7,9 @@ //! was evicted in between -- e.g. by an RBF replacement -- the second lookup //! could come back empty and hit an `.expect("failed loading txos")`. //! -//! This test drives those two separate snapshots directly: SELECT via -//! `Query::lookup_txn()`, then LOOKUP via `Query::mempool().lookup_txos()`. -//! The fix in this MR removes the panic by propagating the miss as a -//! `MissingTxo` error and by holding a single snapshot across both steps in -//! the REST handlers themselves. +//! This test verifies both parts of the fix: a writer cannot enter while the +//! REST-style read snapshot spans SELECT and LOOKUP, and a lookup performed +//! after an explicit eviction returns `MissingTxo` instead of panicking. //! //! Rather than racing real wall-clock timing against electrs's background //! mempool-sync thread (which only runs on a fixed poll interval and makes @@ -75,10 +73,11 @@ fn prevout_lookup_survives_ancestor_eviction() -> Result<()> { let query = tester.query(); - // SELECT: mirrors the GET /tx/:txid handler's `query.lookup_txn()` -- - // one mempool snapshot, cloning out the child tx. - let child_tx = query - .lookup_txn(&child_txid)? + // Mirror the atomic section in GET /tx/:txid: keep the same read guard + // from transaction selection through prevout resolution. + let mempool = query.mempool(); + let child_tx = mempool + .lookup_txn(&child_txid) .expect("child tx should be indexed before eviction"); let outpoints: BTreeSet = child_tx .input @@ -87,11 +86,15 @@ fn prevout_lookup_survives_ancestor_eviction() -> Result<()> { .collect(); assert!(outpoints.contains(&OutPoint::new(parent_txid, parent_vout as u32))); - // Sanity check: resolving prevouts works fine right now, before any - // eviction -- proves the graceful error below is caused by the - // eviction, not by some unrelated test-setup mistake. - let prevouts_before = query.mempool().lookup_txos(outpoints.clone())?; + // A writer that represents the mempool updater must not enter between + // selection and prevout lookup. `try_write` makes the assertion + // deterministic without racing the updater's normal polling interval. + assert!(tester.mempool().try_write().is_err()); + + let prevouts_before = mempool.lookup_txos(outpoints.clone())?; assert!(prevouts_before.contains_key(&OutPoint::new(parent_txid, parent_vout as u32))); + drop(mempool); + assert!(tester.mempool().try_write().is_ok()); // EVICT: RBF-replace the parent on the node by directly double-spending // its inputs (not via `bumpfee`, which refuses once the wallet sees a From 383e5dfbc5f98809aa90255432d8fe143589231b Mon Sep 17 00:00:00 2001 From: EddieHouston Date: Fri, 28 Aug 2026 13:05:18 +0200 Subject: [PATCH 05/28] fix(query): recover from a poisoned mempool lock on the write path --- src/new_index/query.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/new_index/query.rs b/src/new_index/query.rs index 12c4f4d49..9b1f8c8ca 100644 --- a/src/new_index/query.rs +++ b/src/new_index/query.rs @@ -83,7 +83,7 @@ impl Query { let _ = self .mempool .write() - .unwrap() + .unwrap_or_else(|e| e.into_inner()) .add_by_txid(&self.daemon, txid); Ok(txid) } @@ -103,7 +103,7 @@ impl Query { let _ = self .mempool .write() - .unwrap() + .unwrap_or_else(|e| e.into_inner()) .add_by_txids(&self.daemon, &accepted_txids); } Ok(result) From 46e37aec03e94395244ee71edbc7633cafee8445 Mon Sep 17 00:00:00 2001 From: EddieHouston Date: Mon, 14 Sep 2026 15:27:08 +0200 Subject: [PATCH 06/28] fix(rest): preserve mempool snapshots across confirmation and eviction Capture mempool transactions and prevouts before chain reads, then release the lock before response preparation. Deduplicate confirmed history entries and preserve transaction confirmation status. Exercise REST preparation in regression tests and fix Liquid test compatibility. Validation: 111 Bitcoin tests and 112 Liquid tests passed. --- src/lib.rs | 4 + src/new_index/mempool.rs | 3 +- src/rest.rs | 206 ++++++++++++++++------------- src/rest/mempool_tests.rs | 234 +++++++++++++++++++++++++++++++++ tests/common.rs | 4 + tests/mempool_eviction_race.rs | 168 ----------------------- 6 files changed, 362 insertions(+), 257 deletions(-) create mode 100644 src/rest/mempool_tests.rs delete mode 100644 tests/mempool_eviction_race.rs diff --git a/src/lib.rs b/src/lib.rs index af632d973..5cfa08944 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,6 +15,10 @@ extern crate serde_json; #[macro_use] extern crate lazy_static; +// Reuse the integration-test node fixture in private REST unit tests. +#[cfg(test)] +extern crate self as electrs; + pub mod chain; pub mod config; pub mod daemon; diff --git a/src/new_index/mempool.rs b/src/new_index/mempool.rs index ddb91fe9c..08a7d2379 100644 --- a/src/new_index/mempool.rs +++ b/src/new_index/mempool.rs @@ -464,7 +464,8 @@ impl Mempool { Ok(()) } - fn lookup_txo(&self, outpoint: &OutPoint) -> Option { + /// Look up a local mempool output without chain or daemon IO. + pub(crate) fn lookup_txo(&self, outpoint: &OutPoint) -> Option { self.txstore .get(&outpoint.txid) .and_then(|tx| tx.output.get(outpoint.vout as usize).cloned()) diff --git a/src/rest.rs b/src/rest.rs index 1775b635d..011718572 100644 --- a/src/rest.rs +++ b/src/rest.rs @@ -4,7 +4,7 @@ use crate::chain::{ }; use crate::config::Config; use crate::errors; -use crate::new_index::{compute_script_hash, Query, SpendingInput, Utxo}; +use crate::new_index::{compute_script_hash, Mempool, Query, SpendingInput, Utxo}; #[cfg(feature = "liquid")] use crate::util::optional_value_for_newer_blocks; use crate::util::{ @@ -487,6 +487,87 @@ fn ttl_by_depth(height: Option, query: &Query) -> u32 { }) } +// Own the mempool data needed by a response. Chain IO and response construction +// happen after capture returns and releases the read guard. +struct MempoolTxs { + txs: Vec<(Transaction, Option)>, + prevouts: HashMap, +} + +impl MempoolTxs { + fn capture(query: &Query, select: impl FnOnce(&Mempool) -> Vec) -> Self { + let mempool = query.mempool(); + let txs: Vec<_> = select(&mempool).into_iter().map(|tx| (tx, None)).collect(); + let prevouts = txs + .iter() + .flat_map(|(tx, _)| &tx.input) + .filter(|txin| has_prevout(txin)) + .filter_map(|txin| { + mempool + .lookup_txo(&txin.previous_output) + .map(|txout| (txin.previous_output, txout)) + }) + .collect(); + Self { txs, prevouts } + } + + fn prepare(self, query: &Query, config: &Config) -> Result, HttpError> { + let mut prevouts = self.prevouts; + prepare_txs( + self.txs, + |mut outpoints| { + outpoints.retain(|outpoint| !prevouts.contains_key(outpoint)); + prevouts.extend(query.chain().lookup_txos(outpoints)?); + Ok(prevouts) + }, + config, + ) + } +} + +fn prepare_history( + query: &Query, + config: &Config, + mempool_history: impl FnOnce(&Mempool) -> Vec, + chain_history: impl FnOnce() -> errors::Result>, +) -> Result, HttpError> { + // Read mempool first: a transaction confirmed between these reads remains + // in at least one result. Prefer the confirmed entry when both contain it. + let mut snapshot = MempoolTxs::capture(query, mempool_history); + let chain_txs = chain_history().map_err(HttpError::lookup)?; + let confirmed: BTreeSet<_> = chain_txs.iter().map(|(tx, _)| tx.compute_txid()).collect(); + snapshot + .txs + .retain(|(tx, _)| !confirmed.contains(&tx.compute_txid())); + snapshot.txs.extend( + chain_txs + .into_iter() + .map(|(tx, blockid)| (tx, Some(blockid))), + ); + snapshot.prepare(query, config) +} + +fn prepare_tx( + mut snapshot: MempoolTxs, + hash: &Txid, + query: &Query, + config: &Config, +) -> Result<(TransactionValue, u32), HttpError> { + if snapshot.txs.is_empty() { + let tx = query + .chain() + .lookup_txn(hash, None) + .map_err(HttpError::lookup)? + .ok_or_else(|| HttpError::not_found("Transaction not found".to_string()))?; + snapshot.txs.push((tx, None)); + } + // Look up status after transaction selection, including the mempool path. + let blockid = query.chain().tx_confirming_block(hash); + snapshot.txs[0].1 = blockid; + let ttl = ttl_by_depth(blockid.map(|b| b.height), query); + Ok((snapshot.prepare(query, config)?.remove(0), ttl)) +} + fn prepare_txs( txs: Vec<(Transaction, Option)>, lookup_txos: F, @@ -917,27 +998,16 @@ fn handle_blocking_request( None, ) => { let script_hash = to_scripthash(script_type, script_str, config.network_type)?; - // Read outside the mempool guard, so a tx that is both indexed as confirmed - // and still in the local mempool can appear in chain_txs and the mempool - // history below. - let chain_txs = query - .chain() - .history(&script_hash[..], None, CHAIN_TXS_PER_PAGE) - .map_err(HttpError::lookup)?; - - let mempool = query.mempool(); - let mut txs = mempool - .history(&script_hash[..], MAX_MEMPOOL_TXS) - .into_iter() - .map(|tx| (tx, None)) - .collect::>(); - txs.extend( - chain_txs - .into_iter() - .map(|(tx, blockid)| (tx, Some(blockid))), - ); - let txs = prepare_txs(txs, |outpoints| mempool.lookup_txos(outpoints), config)?; - drop(mempool); + let txs = prepare_history( + query, + config, + |mempool| mempool.history(&script_hash[..], MAX_MEMPOOL_TXS), + || { + query + .chain() + .history(&script_hash[..], None, CHAIN_TXS_PER_PAGE) + }, + )?; json_response(txs, TTL_SHORT) } @@ -994,14 +1064,10 @@ fn handle_blocking_request( None, ) => { let script_hash = to_scripthash(script_type, script_str, config.network_type)?; - let mempool = query.mempool(); - let txs = mempool - .history(&script_hash[..], MAX_MEMPOOL_TXS) - .into_iter() - .map(|tx| (tx, None)) - .collect(); - let txs = prepare_txs(txs, |outpoints| mempool.lookup_txos(outpoints), config)?; - drop(mempool); + let txs = MempoolTxs::capture(query, |mempool| { + mempool.history(&script_hash[..], MAX_MEMPOOL_TXS) + }) + .prepare(query, config)?; json_response(txs, TTL_SHORT) } @@ -1040,35 +1106,10 @@ fn handle_blocking_request( } (&Method::GET, Some(&"tx"), Some(hash), None, None, None) => { let hash = Txid::from_str(hash)?; - let chain_tx = query - .chain() - .lookup_txn(&hash, None) - .map_err(HttpError::lookup)?; - let blockid = query.chain().tx_confirming_block(&hash); - let ttl = ttl_by_depth(blockid.as_ref().map(|b| b.height), query); - - let tx = match chain_tx { - Some(tx) => prepare_txs( - vec![(tx, blockid)], - |outpoints| query.chain().lookup_txos(outpoints), - config, - )? - .remove(0), - None => { - let mempool = query.mempool(); - let tx = mempool - .lookup_txn(&hash) - .ok_or_else(|| HttpError::not_found("Transaction not found".to_string()))?; - let prepared = prepare_txs( - vec![(tx, None)], - |outpoints| mempool.lookup_txos(outpoints), - config, - )? - .remove(0); - drop(mempool); - prepared - } - }; + let snapshot = MempoolTxs::capture(query, |mempool| { + mempool.lookup_txn(&hash).into_iter().collect() + }); + let (tx, ttl) = prepare_tx(snapshot, &hash, query, config)?; json_response(tx, ttl) } @@ -1294,27 +1335,16 @@ fn handle_blocking_request( #[cfg(feature = "liquid")] (&Method::GET, Some(&"asset"), Some(asset_str), Some(&"txs"), None, None) => { let asset_id = AssetId::from_str(asset_str)?; - // Read outside the mempool guard, so a tx that is both indexed as confirmed - // and still in the local mempool can appear in chain_txs and the mempool - // history below. - let chain_txs = query - .chain() - .asset_history(&asset_id, None, CHAIN_TXS_PER_PAGE) - .map_err(HttpError::lookup)?; - - let mempool = query.mempool(); - let mut txs = mempool - .asset_history(&asset_id, MAX_MEMPOOL_TXS) - .into_iter() - .map(|tx| (tx, None)) - .collect::>(); - txs.extend( - chain_txs - .into_iter() - .map(|(tx, blockid)| (tx, Some(blockid))), - ); - let txs = prepare_txs(txs, |outpoints| mempool.lookup_txos(outpoints), config)?; - drop(mempool); + let txs = prepare_history( + query, + config, + |mempool| mempool.asset_history(&asset_id, MAX_MEMPOOL_TXS), + || { + query + .chain() + .asset_history(&asset_id, None, CHAIN_TXS_PER_PAGE) + }, + )?; json_response(txs, TTL_SHORT) } @@ -1347,14 +1377,10 @@ fn handle_blocking_request( #[cfg(feature = "liquid")] (&Method::GET, Some(&"asset"), Some(asset_str), Some(&"txs"), Some(&"mempool"), None) => { let asset_id = AssetId::from_str(asset_str)?; - let mempool = query.mempool(); - let txs = mempool - .asset_history(&asset_id, MAX_MEMPOOL_TXS) - .into_iter() - .map(|tx| (tx, None)) - .collect(); - let txs = prepare_txs(txs, |outpoints| mempool.lookup_txos(outpoints), config)?; - drop(mempool); + let txs = MempoolTxs::capture(query, |mempool| { + mempool.asset_history(&asset_id, MAX_MEMPOOL_TXS) + }) + .prepare(query, config)?; json_response(txs, TTL_SHORT) } @@ -1891,3 +1917,7 @@ mod tests { } } } + +#[cfg(test)] +#[path = "rest/mempool_tests.rs"] +mod mempool_tests; diff --git a/src/rest/mempool_tests.rs b/src/rest/mempool_tests.rs new file mode 100644 index 000000000..1eb04ae26 --- /dev/null +++ b/src/rest/mempool_tests.rs @@ -0,0 +1,234 @@ +//! Deterministic races through the private preparation functions used by REST. +use super::*; + +#[allow(dead_code)] +mod common { + include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/common.rs")); +} +use common::Result; + +#[test] +#[cfg(not(feature = "liquid"))] +fn prevout_lookup_survives_ancestor_eviction() -> Result<()> { + let mut tester = common::TestRunner::new()?; + + // Parent: a wallet spend, left unconfirmed. + let addr_parent = tester.newaddress()?; + let parent_txid = tester.send(&addr_parent, "1.0 BTC".parse().unwrap())?; + + let parent_tx = tester.get_raw_transaction(parent_txid)?; + let parent_spk = addr_parent.script_pubkey(); + let (parent_vout, parent_txout) = parent_tx + .output + .iter() + .enumerate() + .find(|(_, o)| o.script_pubkey == parent_spk) + .expect("funding output not found in parent tx"); + + // Child: explicitly spends the parent's own unconfirmed output, so the + // ancestor relationship (and thus the prevout dependency) is guaranteed. + let addr_child = tester.newaddress()?; + let child_value = parent_txout.value - bitcoin::Amount::from_sat(1000); + let raw_child: String = tester.node_client().call( + "createrawtransaction", + &[ + json!([{ "txid": parent_txid.to_string(), "vout": parent_vout }]), + json!({ addr_child.to_string(): child_value.to_btc() }), + ], + )?; + let signed: serde_json::Value = tester + .node_client() + .call("signrawtransactionwithwallet", &[json!(raw_child)])?; + assert_eq!(signed["complete"].as_bool(), Some(true)); + let child_txid_str: String = tester + .node_client() + .call("sendrawtransaction", &[signed["hex"].clone()])?; + let child_txid = child_txid_str.parse().unwrap(); + + tester.sync()?; + + let query = Arc::clone(tester.query()); + let config = Arc::clone(tester.config()); + let snapshot = MempoolTxs::capture(&query, |mempool| { + assert!(tester.mempool().try_write().is_err()); + vec![mempool.lookup_txn(&child_txid).unwrap()] + }); + let child_tx = snapshot.txs[0].0.clone(); + let parent_outpoint = OutPoint::new(parent_txid, parent_vout as u32); + assert!(snapshot.prevouts.contains_key(&parent_outpoint)); + assert!(tester.mempool().try_write().is_ok()); + + // EVICT: RBF-replace the parent on the node by directly double-spending + // its inputs (not via `bumpfee`, which refuses once the wallet sees a + // real descendant -- exactly the descendant we just created). This also + // invalidates and evicts the child, since its input no longer exists. + // Then apply that eviction to electrs's local mempool view -- exactly + // what the background sync loop does on its next tick, just called + // directly and synchronously here instead of racing its timer. + let gettx: serde_json::Value = tester + .node_client() + .call("gettransaction", &[json!(parent_txid.to_string())])?; + let parent_fee = bitcoin::Amount::from_btc(gettx["fee"].as_f64().unwrap().abs()).unwrap(); + let total_output: bitcoin::Amount = parent_tx.output.iter().map(|o| o.value).sum(); + let total_input = total_output + parent_fee; + let higher_fee = bitcoin::Amount::from_sat(50_000); // well above the parent+child's combined fee + let replacement_value = total_input - higher_fee; + + let addr_conflict = tester.newaddress()?; + let raw_conflict: String = tester.node_client().call( + "createrawtransaction", + &[ + json!(parent_tx + .input + .iter() + .map(|txin| json!({ + "txid": txin.previous_output.txid.to_string(), + "vout": txin.previous_output.vout, + })) + .collect::>()), + json!({ addr_conflict.to_string(): replacement_value.to_btc() }), + ], + )?; + let signed_conflict: serde_json::Value = tester + .node_client() + .call("signrawtransactionwithwallet", &[json!(raw_conflict)])?; + assert_eq!(signed_conflict["complete"].as_bool(), Some(true)); + let _replacement_txid: String = tester + .node_client() + .call("sendrawtransaction", &[signed_conflict["hex"].clone()])?; + + let tip = tester.get_best_block_hash()?; + assert!(Mempool::update(&tester.mempool(), &tester.daemon(), &tip)?); + + // Exercise the actual REST preparation path after the ancestor is gone. + // Keep a writer guard during preparation: no mempool re-read is allowed. + let writer = tester.mempool().try_write().unwrap(); + let prepared = snapshot.prepare(&query, &config).unwrap(); + assert_eq!(prepared[0].txid, child_txid); + assert_eq!(prepared[0].fee, 1000); + let body = serde_json::to_value(&prepared[0])?; + assert_eq!( + body["vin"][0]["prevout"]["value"], + parent_txout.value.to_sat() + ); + + // Losing the captured prevout must reach REST's 404 mapping, not abort. + let stale = MempoolTxs { + txs: vec![(child_tx, None)], + prevouts: HashMap::new(), + }; + let error = stale.prepare(&query, &config).err().unwrap(); + assert_eq!(error.0, StatusCode::NOT_FOUND); + drop(writer); + Ok(()) +} + +#[test] +fn history_keeps_transaction_confirmed_between_reads() -> Result<()> { + for confirm_before_chain_read in [false, true] { + let mut tester = common::TestRunner::new()?; + let address = tester.newaddress()?; + let txid = tester.send(&address, "1 BTC".parse().unwrap())?; + let script_hash = compute_script_hash(&address.script_pubkey()); + let query = Arc::clone(tester.query()); + let config = Arc::clone(tester.config()); + let txs = prepare_history( + &query, + &config, + |mempool| mempool.history(&script_hash, MAX_MEMPOOL_TXS), + || { + assert!(tester.mempool().try_write().is_ok()); + if confirm_before_chain_read { + tester.mine().unwrap(); + } + let txs = query + .chain() + .history(&script_hash, None, CHAIN_TXS_PER_PAGE); + if !confirm_before_chain_read { + // A chain-first implementation loses this transaction when + // it subsequently reads the now-empty mempool. + tester.mine().unwrap(); + } + txs + }, + ) + .unwrap(); + let matches: Vec<_> = txs.iter().filter(|tx| tx.txid == txid).collect(); + assert_eq!(matches.len(), 1); + assert_eq!( + matches[0].status.as_ref().unwrap().confirmed, + confirm_before_chain_read + ); + } + Ok(()) +} + +#[test] +fn selected_mempool_transaction_uses_current_confirmation_status() -> Result<()> { + let mut tester = common::TestRunner::new()?; + let address = tester.newaddress()?; + let txid = tester.send(&address, "1 BTC".parse().unwrap())?; + let query = Arc::clone(tester.query()); + let config = Arc::clone(tester.config()); + let snapshot = MempoolTxs::capture(&query, |mempool| vec![mempool.lookup_txn(&txid).unwrap()]); + let blockhash = tester.mine()?; + let writer = tester.mempool().try_write().unwrap(); + let (tx, ttl) = prepare_tx(snapshot, &txid, &query, &config).unwrap(); + let status = tx.status.unwrap(); + assert!(status.confirmed); + assert_eq!(status.block_hash, Some(blockhash)); + assert_eq!(ttl, ttl_by_depth(status.block_height, &query)); + drop(writer); + + // The same path must also find a transaction already removed from mempool. + let snapshot = MempoolTxs::capture(&query, |mempool| { + mempool.lookup_txn(&txid).into_iter().collect() + }); + assert!(snapshot.txs.is_empty()); + let (tx, _) = prepare_tx(snapshot, &txid, &query, &config).unwrap(); + assert_eq!(tx.status.unwrap().block_hash, Some(blockhash)); + Ok(()) +} + +#[test] +#[cfg(feature = "liquid")] +fn asset_history_keeps_transaction_confirmed_between_reads() -> Result<()> { + use elementsd::bitcoincore_rpc::RpcApi; + for confirm_before_chain_read in [false, true] { + let mut tester = common::TestRunner::new()?; + let issued: serde_json::Value = tester + .node_client() + .call("issueasset", &[json!(1), json!(0)])?; + let txid: Txid = issued["txid"].as_str().unwrap().parse().unwrap(); + let asset: AssetId = issued["asset"].as_str().unwrap().parse().unwrap(); + tester.sync()?; + let query = Arc::clone(tester.query()); + let config = Arc::clone(tester.config()); + let txs = prepare_history( + &query, + &config, + |mempool| mempool.asset_history(&asset, MAX_MEMPOOL_TXS), + || { + assert!(tester.mempool().try_write().is_ok()); + if confirm_before_chain_read { + tester.mine().unwrap(); + } + let txs = query + .chain() + .asset_history(&asset, None, CHAIN_TXS_PER_PAGE); + if !confirm_before_chain_read { + tester.mine().unwrap(); + } + txs + }, + ) + .unwrap(); + let matches: Vec<_> = txs.iter().filter(|tx| tx.txid == txid).collect(); + assert_eq!(matches.len(), 1); + assert_eq!( + matches[0].status.as_ref().unwrap().confirmed, + confirm_before_chain_read + ); + } + Ok(()) +} diff --git a/tests/common.rs b/tests/common.rs index c3f479050..ba84724af 100644 --- a/tests/common.rs +++ b/tests/common.rs @@ -227,6 +227,10 @@ impl TestRunner { return &self.node.client(); } + pub fn config(&self) -> &Arc { + &self.config + } + pub fn query(&self) -> &Arc { &self.query } diff --git a/tests/mempool_eviction_race.rs b/tests/mempool_eviction_race.rs deleted file mode 100644 index 79d0c29aa..000000000 --- a/tests/mempool_eviction_race.rs +++ /dev/null @@ -1,168 +0,0 @@ -//! Deterministic reproduction of the mempool-eviction prevout-lookup abort -//! in src/new_index/query.rs and src/rest.rs. -//! -//! The REST handlers for e.g. GET /tx/:txid used to look up a transaction -//! (once, in one mempool snapshot) and then separately resolve its prevouts -//! via a second, later mempool snapshot. If the referenced mempool ancestor -//! was evicted in between -- e.g. by an RBF replacement -- the second lookup -//! could come back empty and hit an `.expect("failed loading txos")`. -//! -//! This test verifies both parts of the fix: a writer cannot enter while the -//! REST-style read snapshot spans SELECT and LOOKUP, and a lookup performed -//! after an explicit eviction returns `MissingTxo` instead of panicking. -//! -//! Rather than racing real wall-clock timing against electrs's background -//! mempool-sync thread (which only runs on a fixed poll interval and makes -//! the window practically unhittable black-box), this test drives the same -//! two steps directly and deterministically: select the transaction, then -//! explicitly evict its ancestor by replacing it on the node and calling -//! `Mempool::update()` inline (this is exactly what the background sync -//! loop does, just invoked synchronously instead of on a timer), then -//! resolve prevouts. No timing luck involved. - -use std::collections::BTreeSet; - -use serde_json::json; - -use electrs::chain::OutPoint; -use electrs::errors::ErrorKind; -use electrs::new_index::Mempool; - -pub mod common; - -use common::Result; - -#[test] -fn prevout_lookup_survives_ancestor_eviction() -> Result<()> { - let mut tester = common::TestRunner::new()?; - - // Parent: a wallet spend, left unconfirmed. - let addr_parent = tester.newaddress()?; - let parent_txid = tester.send(&addr_parent, "1.0 BTC".parse().unwrap())?; - - let parent_tx = tester.get_raw_transaction(parent_txid)?; - let parent_spk = addr_parent.script_pubkey(); - let (parent_vout, parent_txout) = parent_tx - .output - .iter() - .enumerate() - .find(|(_, o)| o.script_pubkey == parent_spk) - .expect("funding output not found in parent tx"); - - // Child: explicitly spends the parent's own unconfirmed output, so the - // ancestor relationship (and thus the prevout dependency) is guaranteed. - let addr_child = tester.newaddress()?; - let child_value = parent_txout.value - bitcoin::Amount::from_sat(1000); - let raw_child: String = tester.node_client().call( - "createrawtransaction", - &[ - json!([{ "txid": parent_txid.to_string(), "vout": parent_vout }]), - json!({ addr_child.to_string(): child_value.to_btc() }), - ], - )?; - let signed: serde_json::Value = tester - .node_client() - .call("signrawtransactionwithwallet", &[json!(raw_child)])?; - assert_eq!(signed["complete"].as_bool(), Some(true)); - let child_txid_str: String = tester - .node_client() - .call("sendrawtransaction", &[signed["hex"].clone()])?; - let child_txid = child_txid_str.parse().unwrap(); - - tester.sync()?; - - let query = tester.query(); - - // Mirror the atomic section in GET /tx/:txid: keep the same read guard - // from transaction selection through prevout resolution. - let mempool = query.mempool(); - let child_tx = mempool - .lookup_txn(&child_txid) - .expect("child tx should be indexed before eviction"); - let outpoints: BTreeSet = child_tx - .input - .iter() - .map(|txin| txin.previous_output) - .collect(); - assert!(outpoints.contains(&OutPoint::new(parent_txid, parent_vout as u32))); - - // A writer that represents the mempool updater must not enter between - // selection and prevout lookup. `try_write` makes the assertion - // deterministic without racing the updater's normal polling interval. - assert!(tester.mempool().try_write().is_err()); - - let prevouts_before = mempool.lookup_txos(outpoints.clone())?; - assert!(prevouts_before.contains_key(&OutPoint::new(parent_txid, parent_vout as u32))); - drop(mempool); - assert!(tester.mempool().try_write().is_ok()); - - // EVICT: RBF-replace the parent on the node by directly double-spending - // its inputs (not via `bumpfee`, which refuses once the wallet sees a - // real descendant -- exactly the descendant we just created). This also - // invalidates and evicts the child, since its input no longer exists. - // Then apply that eviction to electrs's local mempool view -- exactly - // what the background sync loop does on its next tick, just called - // directly and synchronously here instead of racing its timer. - let gettx: serde_json::Value = tester - .node_client() - .call("gettransaction", &[json!(parent_txid.to_string())])?; - let parent_fee = bitcoin::Amount::from_btc(gettx["fee"].as_f64().unwrap().abs()).unwrap(); - let total_output: bitcoin::Amount = parent_tx.output.iter().map(|o| o.value).sum(); - let total_input = total_output + parent_fee; - let higher_fee = bitcoin::Amount::from_sat(50_000); // well above the parent+child's combined fee - let replacement_value = total_input - higher_fee; - - let addr_conflict = tester.newaddress()?; - let raw_conflict: String = tester.node_client().call( - "createrawtransaction", - &[ - json!(parent_tx - .input - .iter() - .map(|txin| json!({ - "txid": txin.previous_output.txid.to_string(), - "vout": txin.previous_output.vout, - })) - .collect::>()), - json!({ addr_conflict.to_string(): replacement_value.to_btc() }), - ], - )?; - let signed_conflict: serde_json::Value = tester - .node_client() - .call("signrawtransactionwithwallet", &[json!(raw_conflict)])?; - assert_eq!(signed_conflict["complete"].as_bool(), Some(true)); - let _replacement_txid: String = tester - .node_client() - .call("sendrawtransaction", &[signed_conflict["hex"].clone()])?; - - let tip = tester.get_best_block_hash()?; - assert!(Mempool::update(&tester.mempool(), &tester.daemon(), &tip)?); - - // LOOKUP: a second, independent mempool snapshot on the *same* outpoints - // selected before the eviction -- the parent's output is now gone from - // both the local mempool view (evicted) and the confirmed chain (it was - // never mined). Pre-fix this panicked at the `.expect("failed loading - // txos")` in the `Query::lookup_txos` wrapper; post-fix it must come - // back as a graceful `MissingTxo` error instead. - let result = query.mempool().lookup_txos(outpoints); - - match result { - Ok(txos) => panic!( - "expected lookup_txos() to report the evicted prevout as missing, \ - but it returned successfully with {} txos -- the race wasn't \ - reproduced (check the eviction actually happened)", - txos.len() - ), - Err(ref e) => match e.kind() { - ErrorKind::MissingTxo(outpoint) => { - assert_eq!( - outpoint, - &OutPoint::new(parent_txid, parent_vout as u32).to_string()[..] - ); - } - other => panic!("expected ErrorKind::MissingTxo, got: {:?}", other), - }, - } - - Ok(()) -} From aa2fc45f4e315d3bfda1a7d0e50094bdd9e42d0d Mon Sep 17 00:00:00 2001 From: EddieHouston Date: Tue, 15 Sep 2026 14:38:04 +0200 Subject: [PATCH 07/28] fix(rest): avoid mempool contention and classify lookup failures Check chain storage before capturing mempool transactions. Return 400 for invalid block page indices, 503 for storage and confirmed-prevout failures, and 404 for vanished unconfirmed mempool prevouts. --- src/new_index/db.rs | 4 ++ src/new_index/schema.rs | 82 +++++++++++++++++++++++- src/rest.rs | 130 ++++++++++++++++++++++++++++++++------ src/rest/mempool_tests.rs | 43 ++++++++++++- 4 files changed, 234 insertions(+), 25 deletions(-) diff --git a/src/new_index/db.rs b/src/new_index/db.rs index 73193fc7f..36911fd4e 100644 --- a/src/new_index/db.rs +++ b/src/new_index/db.rs @@ -360,6 +360,10 @@ impl DB { self.db.get(key).unwrap().map(|v| v.to_vec()) } + pub fn try_get(&self, key: &[u8]) -> Result, rocksdb::Error> { + self.db.get(key).map(|value| value.map(|v| v.to_vec())) + } + pub fn multi_get(&self, keys: I) -> Vec>, rocksdb::Error>> where K: AsRef<[u8]>, diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index fead9decb..f01a6b303 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -667,14 +667,35 @@ impl ChainQuery { } } + pub fn try_get_block_txids(&self, hash: &BlockHash) -> Result>> { + let _timer = self.start_timer("get_block_txids"); + if self.light_mode { + let mut blockinfo = match self.daemon.getblock_raw(hash, 1) { + Ok(blockinfo) => blockinfo, + Err(_) => return Ok(None), + }; + Ok(Some(serde_json::from_value(blockinfo["tx"].take()) + .chain_err(|| "failed to parse block txids")?)) + } else { + self.store + .txstore_db + .try_get(&BlockRow::txids_key(full_hash(&hash[..])))? + .map(|val| bincode::deserialize_little(&val).chain_err(|| "failed to parse block txids")) + .transpose() + } + } + pub fn get_block_txs( &self, hash: &BlockHash, start_index: usize, limit: usize, ) -> Result> { - let txids = self.get_block_txids(hash).chain_err(|| "block not found")?; - ensure!(start_index < txids.len(), "start index out of range"); + let txids = self.try_get_block_txids(hash)?.chain_err(|| "block not found")?; + ensure!( + start_index < txids.len(), + ErrorKind::InvalidParams("start index out of range".to_string()) + ); let txids_with_blockhash = txids .into_iter() @@ -1149,7 +1170,7 @@ impl ChainQuery { blockhash: Option<&BlockHash>, ) -> Result> { let _timer = self.start_timer("lookup_txn"); - self.lookup_raw_txn(txid, blockhash) + self.try_lookup_raw_txn(txid, blockhash)? .map(|rawtx| deserialize(&rawtx).chain_err(|| "failed to parse Transaction")) .transpose() } @@ -1194,6 +1215,28 @@ impl ChainQuery { } } + fn try_lookup_raw_txn(&self, txid: &Txid, blockhash: Option<&BlockHash>) -> Result> { + if self.light_mode { + let queried_blockhash = if blockhash.is_none() { + self.try_tx_confirming_block(txid)?.map(|blockid| blockid.hash) + } else { + None + }; + let blockhash = match blockhash.or(queried_blockhash.as_ref()) { + Some(blockhash) => blockhash, + None => return Ok(None), + }; + let txval = match self.daemon.gettransaction_raw(txid, blockhash, false) { + Ok(txval) => txval, + Err(_) => return Ok(None), + }; + let txhex = txval.as_str().chain_err(|| "invalid transaction RPC response")?; + Ok(Some(Bytes::from_hex(txhex).chain_err(|| "invalid transaction hex")?)) + } else { + Ok(self.store.txstore_db.try_get(&TxRow::key(&txid[..]))?) + } + } + pub fn lookup_txo(&self, outpoint: &OutPoint) -> Option { let _timer = self.start_timer("lookup_txo"); lookup_txo(&self.store.txstore_db, outpoint) @@ -1250,6 +1293,18 @@ impl ChainQuery { Some(headers.header_by_height(height as usize)?.into()) } + pub fn try_tx_confirming_block(&self, txid: &Txid) -> Result> { + let _timer = self.start_timer("tx_confirming_block"); + let row_value = match self.store.history_db.try_get(&TxConfRow::key(txid))? { + Some(row_value) => row_value, + None => return Ok(None), + }; + ensure!(row_value.len() == 4, "invalid transaction confirmation row"); + let height = TxConfRow::height_from_val(&row_value); + let headers = self.store.indexed_headers.read().unwrap(); + Ok(headers.header_by_height(height as usize).map(Into::into)) + } + pub fn lookup_confirmations(&self, txids: BTreeSet) -> HashMap { let _timer = self.start_timer("lookup_confirmations"); let headers = self.store.indexed_headers.read().unwrap(); @@ -2077,6 +2132,27 @@ pub mod bench { mod tests { use super::*; + #[allow(dead_code)] + mod common { + include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/common.rs")); + } + + #[test] + fn corrupt_single_key_rows_return_errors() -> common::Result<()> { + let mut tester = common::TestRunner::new()?; + let address = tester.newaddress()?; + let txid = tester.send(&address, "1 BTC".parse().unwrap())?; + let blockhash = tester.mine()?; + let chain = tester.query().chain(); + + chain.store.txstore_db.put(&BlockRow::txids_key(full_hash(&blockhash[..])), &[]); + assert!(chain.get_block_txs(&blockhash, 0, 25).is_err()); + + chain.store.history_db.put(&TxConfRow::key(&txid), &[]); + assert!(chain.try_tx_confirming_block(&txid).is_err()); + Ok(()) + } + #[test] fn test_completed_prefix_stops_at_first_gap() { let items = [true, true, false, true]; diff --git a/src/rest.rs b/src/rest.rs index 011718572..18023f358 100644 --- a/src/rest.rs +++ b/src/rest.rs @@ -513,11 +513,24 @@ impl MempoolTxs { fn prepare(self, query: &Query, config: &Config) -> Result, HttpError> { let mut prevouts = self.prevouts; + let confirmed_prevouts: BTreeSet<_> = self + .txs + .iter() + .filter(|(_, blockid)| blockid.is_some()) + .flat_map(|(tx, _)| tx.input.iter()) + .filter(|txin| has_prevout(txin)) + .map(|txin| txin.previous_output) + .collect(); prepare_txs( self.txs, |mut outpoints| { outpoints.retain(|outpoint| !prevouts.contains_key(outpoint)); - prevouts.extend(query.chain().lookup_txos(outpoints)?); + prevouts.extend( + query + .chain() + .lookup_txos(outpoints) + .map_err(|err| HttpError::mempool_prevout(err, query, &confirmed_prevouts))?, + ); Ok(prevouts) }, config, @@ -548,6 +561,29 @@ fn prepare_history( } fn prepare_tx( + hash: &Txid, + query: &Query, + config: &Config, +) -> Result<(TransactionValue, u32), HttpError> { + if let Some(tx) = query.chain().lookup_txn(hash, None).map_err(HttpError::lookup)? { + let blockid = query.chain().try_tx_confirming_block(hash).map_err(HttpError::lookup)?; + let ttl = ttl_by_depth(blockid.map(|b| b.height), query); + let tx = prepare_txs( + vec![(tx, blockid)], + |outpoints| query.chain().lookup_txos(outpoints).map_err(HttpError::lookup), + config, + )? + .remove(0); + return Ok((tx, ttl)); + } + + let snapshot = MempoolTxs::capture(query, |mempool| { + mempool.lookup_txn(hash).into_iter().collect() + }); + prepare_captured_tx(snapshot, hash, query, config) +} + +fn prepare_captured_tx( mut snapshot: MempoolTxs, hash: &Txid, query: &Query, @@ -562,7 +598,7 @@ fn prepare_tx( snapshot.txs.push((tx, None)); } // Look up status after transaction selection, including the mempool path. - let blockid = query.chain().tx_confirming_block(hash); + let blockid = query.chain().try_tx_confirming_block(hash).map_err(HttpError::lookup)?; snapshot.txs[0].1 = blockid; let ttl = ttl_by_depth(blockid.map(|b| b.height), query); Ok((snapshot.prepare(query, config)?.remove(0), ttl)) @@ -574,7 +610,7 @@ fn prepare_txs( config: &Config, ) -> Result, HttpError> where - F: FnOnce(BTreeSet) -> errors::Result>, + F: FnOnce(BTreeSet) -> Result, HttpError>, { let outpoints = txs .iter() @@ -586,7 +622,7 @@ where }) .collect(); - let prevouts = lookup_txos(outpoints).map_err(HttpError::lookup)?; + let prevouts = lookup_txos(outpoints)?; Ok(txs .into_iter() @@ -892,7 +928,8 @@ fn handle_blocking_request( let hash = BlockHash::from_str(hash)?; let txids = query .chain() - .get_block_txids(&hash) + .try_get_block_txids(&hash) + .map_err(HttpError::lookup)? .ok_or_else(|| HttpError::not_found("Block not found".to_string()))?; json_response(txids, TTL_LONG) } @@ -925,7 +962,8 @@ fn handle_blocking_request( let index: usize = index.parse()?; let txids = query .chain() - .get_block_txids(&hash) + .try_get_block_txids(&hash) + .map_err(HttpError::lookup)? .ok_or_else(|| HttpError::not_found("Block not found".to_string()))?; if index >= txids.len() { bail!(HttpError::not_found("tx index out of range".to_string())); @@ -955,7 +993,8 @@ fn handle_blocking_request( let txs = query .chain() - .get_block_txs(&hash, start_index, CHAIN_TXS_PER_PAGE)? + .get_block_txs(&hash, start_index, CHAIN_TXS_PER_PAGE) + .map_err(HttpError::block_txs)? .into_iter() .map(|tx| (tx, blockid)) .collect(); @@ -964,7 +1003,11 @@ fn handle_blocking_request( let ttl = ttl_by_depth(blockid.map(|b| b.height), query); json_response( - prepare_txs(txs, |outpoints| query.chain().lookup_txos(outpoints), config)?, + prepare_txs( + txs, + |outpoints| query.chain().lookup_txos(outpoints).map_err(HttpError::lookup), + config, + )?, ttl, ) } @@ -1043,7 +1086,11 @@ fn handle_blocking_request( .collect(); json_response( - prepare_txs(txs, |outpoints| query.chain().lookup_txos(outpoints), config)?, + prepare_txs( + txs, + |outpoints| query.chain().lookup_txos(outpoints).map_err(HttpError::lookup), + config, + )?, TTL_SHORT, ) } @@ -1106,10 +1153,7 @@ fn handle_blocking_request( } (&Method::GET, Some(&"tx"), Some(hash), None, None, None) => { let hash = Txid::from_str(hash)?; - let snapshot = MempoolTxs::capture(query, |mempool| { - mempool.lookup_txn(&hash).into_iter().collect() - }); - let (tx, ttl) = prepare_tx(snapshot, &hash, query, config)?; + let (tx, ttl) = prepare_tx(&hash, query, config)?; json_response(tx, ttl) } @@ -1369,7 +1413,11 @@ fn handle_blocking_request( .collect(); json_response( - prepare_txs(txs, |outpoints| query.chain().lookup_txos(outpoints), config)?, + prepare_txs( + txs, + |outpoints| query.chain().lookup_txos(outpoints).map_err(HttpError::lookup), + config, + )?, TTL_SHORT, ) } @@ -1599,13 +1647,43 @@ impl HttpError { } fn lookup(err: errors::Error) -> Self { - let status = match err.kind() { - errors::ErrorKind::MissingTxo(_) => StatusCode::NOT_FOUND, - _ => StatusCode::SERVICE_UNAVAILABLE, - }; + let status = StatusCode::SERVICE_UNAVAILABLE; warn!("REST lookup failed status='{}' err='{:?}'", status, err); HttpError(status, err.to_string()) } + + fn block_txs(err: errors::Error) -> Self { + match err.kind() { + errors::ErrorKind::InvalidParams(_) => HttpError::from(err.to_string()), + _ => HttpError::lookup(err), + } + } + + fn mempool_prevout( + err: errors::Error, + query: &Query, + confirmed_prevouts: &BTreeSet, + ) -> Self { + if let errors::ErrorKind::MissingTxo(missing) = err.kind() { + let used_by_confirmed_tx = confirmed_prevouts + .iter() + .any(|outpoint| outpoint.to_string() == *missing); + let confirmed_parent = missing + .rsplit_once(':') + .and_then(|(txid, _)| Txid::from_str(txid).ok()) + .map(|txid| query.chain().try_tx_confirming_block(&txid)); + let confirmed_parent = match confirmed_parent { + Some(Ok(blockid)) => blockid.is_some(), + Some(Err(err)) => return HttpError::lookup(err), + None => false, + }; + if !used_by_confirmed_tx && !confirmed_parent { + warn!("REST mempool prevout vanished err='{:?}'", err); + return HttpError::not_found(err.to_string()); + } + } + HttpError::lookup(err) + } } impl From for HttpError { @@ -1757,16 +1835,28 @@ mod tests { } #[test] - fn transaction_preparation_failures_map_to_not_found_or_unavailable() { + fn storage_lookup_failures_are_unavailable() { let missing = HttpError::lookup(errors::Error::from(ErrorKind::MissingTxo( "abc:0".to_string(), ))); - assert_eq!(missing.0, StatusCode::NOT_FOUND); + assert_eq!(missing.0, StatusCode::SERVICE_UNAVAILABLE); let corrupt = HttpError::lookup(errors::Error::from("failed to parse TxOut")); assert_eq!(corrupt.0, StatusCode::SERVICE_UNAVAILABLE); } + #[test] + fn block_page_errors_distinguish_invalid_index_from_storage_failure() { + let invalid = HttpError::block_txs(errors::Error::from(ErrorKind::InvalidParams( + "start index out of range".to_string(), + ))); + assert_eq!(invalid.0, StatusCode::BAD_REQUEST); + assert_eq!(invalid.1, "start index out of range"); + + let storage = HttpError::block_txs(errors::Error::from("missing indexed transaction")); + assert_eq!(storage.0, StatusCode::SERVICE_UNAVAILABLE); + } + #[test] fn test_parse_query_param() { let mut query_params = HashMap::new(); diff --git a/src/rest/mempool_tests.rs b/src/rest/mempool_tests.rs index 1eb04ae26..7adf8e6ac 100644 --- a/src/rest/mempool_tests.rs +++ b/src/rest/mempool_tests.rs @@ -173,7 +173,7 @@ fn selected_mempool_transaction_uses_current_confirmation_status() -> Result<()> let snapshot = MempoolTxs::capture(&query, |mempool| vec![mempool.lookup_txn(&txid).unwrap()]); let blockhash = tester.mine()?; let writer = tester.mempool().try_write().unwrap(); - let (tx, ttl) = prepare_tx(snapshot, &txid, &query, &config).unwrap(); + let (tx, ttl) = prepare_captured_tx(snapshot, &txid, &query, &config).unwrap(); let status = tx.status.unwrap(); assert!(status.confirmed); assert_eq!(status.block_hash, Some(blockhash)); @@ -185,11 +185,50 @@ fn selected_mempool_transaction_uses_current_confirmation_status() -> Result<()> mempool.lookup_txn(&txid).into_iter().collect() }); assert!(snapshot.txs.is_empty()); - let (tx, _) = prepare_tx(snapshot, &txid, &query, &config).unwrap(); + let (tx, _) = prepare_captured_tx(snapshot, &txid, &query, &config).unwrap(); assert_eq!(tx.status.unwrap().block_hash, Some(blockhash)); Ok(()) } +#[test] +fn confirmed_transaction_does_not_wait_for_mempool_writer() -> Result<()> { + use std::sync::mpsc; + use std::time::Duration; + + let mut tester = common::TestRunner::new()?; + let address = tester.newaddress()?; + let txid = tester.send(&address, "1 BTC".parse().unwrap())?; + tester.mine()?; + + let query = Arc::clone(tester.query()); + let config = Arc::clone(tester.config()); + let writer = tester.mempool().try_write().unwrap(); + let (sender, receiver) = mpsc::channel(); + let worker = std::thread::spawn(move || { + let result = prepare_tx(&txid, &query, &config).map(|(tx, _)| tx.status.unwrap().confirmed); + sender.send(result).unwrap(); + }); + + let completed = receiver.recv_timeout(Duration::from_secs(5)); + drop(writer); + worker.join().unwrap(); + assert!(completed.unwrap().unwrap()); + Ok(()) +} + +#[test] +fn missing_confirmed_parent_is_unavailable() -> Result<()> { + let mut tester = common::TestRunner::new()?; + let address = tester.newaddress()?; + let parent = tester.send(&address, "1 BTC".parse().unwrap())?; + tester.mine()?; + + let missing = errors::Error::from(errors::ErrorKind::MissingTxo(format!("{}:0", parent))); + let status = HttpError::mempool_prevout(missing, tester.query(), &BTreeSet::new()).0; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); + Ok(()) +} + #[test] #[cfg(feature = "liquid")] fn asset_history_keeps_transaction_confirmed_between_reads() -> Result<()> { From 61cd9397de54c46bab335242e19660f6b3876154 Mon Sep 17 00:00:00 2001 From: Edward Houston Date: Wed, 16 Sep 2026 11:21:34 +0200 Subject: [PATCH 08/28] test: share node fixture across unit tests --- src/lib.rs | 6 ++++++ src/new_index/schema.rs | 6 +----- src/rest/mempool_tests.rs | 8 ++------ 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 5cfa08944..1d6fb56e0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,6 +19,12 @@ extern crate lazy_static; #[cfg(test)] extern crate self as electrs; +#[cfg(test)] +#[allow(dead_code)] +mod test_common { + include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/common.rs")); +} + pub mod chain; pub mod config; pub mod daemon; diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index f01a6b303..95b1224ba 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -2131,11 +2131,7 @@ pub mod bench { #[cfg(test)] mod tests { use super::*; - - #[allow(dead_code)] - mod common { - include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/common.rs")); - } + use crate::test_common as common; #[test] fn corrupt_single_key_rows_return_errors() -> common::Result<()> { diff --git a/src/rest/mempool_tests.rs b/src/rest/mempool_tests.rs index 7adf8e6ac..051111b29 100644 --- a/src/rest/mempool_tests.rs +++ b/src/rest/mempool_tests.rs @@ -1,11 +1,7 @@ //! Deterministic races through the private preparation functions used by REST. use super::*; - -#[allow(dead_code)] -mod common { - include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/common.rs")); -} -use common::Result; +use crate::test_common as common; +use crate::test_common::Result; #[test] #[cfg(not(feature = "liquid"))] From 2ea021228492649f0129ad61d0be3aee68667869 Mon Sep 17 00:00:00 2001 From: Edward Houston Date: Mon, 28 Sep 2026 12:11:29 +0200 Subject: [PATCH 09/28] test: remove duplicate query method on TestRunner --- tests/common.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/common.rs b/tests/common.rs index ba84724af..af21d0f3b 100644 --- a/tests/common.rs +++ b/tests/common.rs @@ -212,10 +212,6 @@ impl TestRunner { }) } - pub fn query(&self) -> Arc { - Arc::clone(&self.query) - } - pub fn metrics_addr(&self) -> net::SocketAddr { self.metrics_addr } From 6d591e2f133de187718c7a71e8aff5d074e8d16c Mon Sep 17 00:00:00 2001 From: Oleksandr Fedorenko Date: Mon, 14 Sep 2026 05:24:57 +0300 Subject: [PATCH 10/28] fix(db): pin index_unspendables and address_search into DB compatibility check --- README.md | 5 +- src/new_index/db.rs | 169 ++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 166 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 5e911aa16..7a5f93e35 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,10 @@ In addition to electrs's original configuration options, a few new options are a - `--http-addr ` - HTTP server address/port to listen on (default: `127.0.0.1:3000`). - `--lightmode` - enable light mode (see above) - `--cors ` - origins allowed to make cross-site request (optional, defaults to none). -- `--address-search` - enables the by-prefix address search index. +- `--address-search` - enables the by-prefix address search index. Best-effort: entries are + written when a block is indexed but are not removed if that block is later reorged out, so + the index may return addresses with no transaction in the current best chain. Run a full + reindex to clear stale entries if this matters for your deployment. - `--index-unspendables` - enables indexing of provably unspendable outputs. - `--enable-mining-rest` - enables cached mining-related HTTP endpoints. - `--utxos-limit ` - maximum number of utxos to return per address. diff --git a/src/new_index/db.rs b/src/new_index/db.rs index 36911fd4e..2ca473dc7 100644 --- a/src/new_index/db.rs +++ b/src/new_index/db.rs @@ -380,15 +380,33 @@ impl DB { } fn verify_compatibility(&self, config: &Config) { - let compatibility_bytes = bincode::serialize_little(&(DB_VERSION, config.light_mode)).unwrap(); + let compatibility_bytes = bincode::serialize_little(&( + DB_VERSION, + config.light_mode, + config.index_unspendables, + config.address_search, + )) + .unwrap(); + + let stored = match self.get(b"V") { + None => return self.put(b"V", &compatibility_bytes), + Some(stored) if stored == compatibility_bytes => return, + Some(stored) => stored, + }; - match self.get(b"V") { - None => self.put(b"V", &compatibility_bytes), - Some(x) if x != compatibility_bytes => { - panic!("Incompatible database found. Please reindex or migrate.") - } - Some(_) => (), + let legacy_bytes = bincode::serialize_little(&(DB_VERSION, config.light_mode)).unwrap(); + if stored == legacy_bytes { + warn!( + "upgrading legacy DB compatibility record using index_unspendables={} and address_search={}; \ + these values must match the settings originally used to build this database, and \ + historical values cannot be verified", + config.index_unspendables, + config.address_search, + ); + return self.put(b"V", &compatibility_bytes); } + + panic!("Incompatible database found. Please reindex or migrate.") } #[cfg(test)] @@ -599,4 +617,141 @@ mod tests { } assert_eq!(count, 9, "expected 9 total rows, got {}", count); } + + fn make_test_config(index_unspendables: bool, address_search: bool, light_mode: bool) -> crate::config::Config { + use crate::config::{Config, RpcLogging}; + use std::path::PathBuf; + + #[cfg(not(feature = "liquid"))] + let network_type = crate::chain::Network::Regtest; + #[cfg(feature = "liquid")] + let network_type = crate::chain::Network::LiquidRegtest; + + let unused_addr = "127.0.0.1:0".parse().unwrap(); + + Config { + log: stderrlog::new(), + network_type, + db_path: PathBuf::from("/tmp/electrs-test-unused"), + daemon_dir: PathBuf::from("/tmp/electrs-test-unused"), + blocks_dir: PathBuf::from("/tmp/electrs-test-unused"), + daemon_rpc_addr: unused_addr, + daemon_rpc_fallback_addr: None, + daemon_parallelism: 1, + daemon_conn_max_age: None, + cookie: None, + electrum_rpc_addr: unused_addr, + electrum_rpc_conn_max_age: None, + electrum_rpc_max_request_num_bytes: 1_048_576, + http_addr: unused_addr, + http_socket_file: None, + monitoring_addr: unused_addr, + jsonrpc_import: false, + light_mode, + address_search, + index_unspendables, + enable_mining_rest: false, + cors: None, + precache_scripts: None, + utxos_limit: 100, + electrum_txs_limit: 100, + electrum_subscription_limit: 10_000, + electrum_checkpoint_proof_concurrency_limit: 2, + electrum_banner: String::new(), + rpc_logging: RpcLogging::default(), + zmq_addr: None, + db_block_cache_mb: 8, + db_parallelism: 1, + db_write_buffer_size_mb: 8, + initial_sync_batch_size: 10, + db_cache_index_filter_blocks: false, + #[cfg(feature = "liquid")] + parent_network: bitcoin::Network::Regtest, + #[cfg(feature = "liquid")] + asset_db_path: None, + } + } + + #[test] + fn test_verify_compatibility_accepts_matching_flags() { + let dir = tempfile::tempdir().unwrap(); + let db = DB::open_test(dir.path()); + + let config = make_test_config(true, false, false); + db.verify_compatibility(&config); + db.verify_compatibility(&config); + } + + #[test] + fn test_verify_compatibility_rejects_index_unspendables_mismatch() { + let dir = tempfile::tempdir().unwrap(); + let db = DB::open_test(dir.path()); + + db.verify_compatibility(&make_test_config(false, false, false)); + + let toggled = make_test_config(true, false, false); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + db.verify_compatibility(&toggled); + })); + assert!( + result.is_err(), + "verify_compatibility should reject a DB reopened with a different index_unspendables flag" + ); + } + + #[test] + fn test_verify_compatibility_rejects_address_search_mismatch() { + let dir = tempfile::tempdir().unwrap(); + let db = DB::open_test(dir.path()); + + db.verify_compatibility(&make_test_config(false, false, false)); + + let toggled = make_test_config(false, true, false); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + db.verify_compatibility(&toggled); + })); + assert!( + result.is_err(), + "verify_compatibility should reject a DB reopened with a different address_search flag" + ); + } + + #[test] + fn test_verify_compatibility_upgrades_legacy_record_without_reindex() { + let dir = tempfile::tempdir().unwrap(); + let db = DB::open_test(dir.path()); + + let legacy_bytes = bincode::serialize_little(&(DB_VERSION, false)).unwrap(); + db.put(b"V", &legacy_bytes); + + let config = make_test_config(true, true, false); + db.verify_compatibility(&config); + + let expected = bincode::serialize_little(&(DB_VERSION, false, true, true)).unwrap(); + assert_eq!(db.get(b"V").unwrap(), expected); + + let toggled = make_test_config(false, true, false); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + db.verify_compatibility(&toggled); + })); + assert!(result.is_err(), "upgraded record should still guard future mismatches"); + } + + #[test] + fn test_verify_compatibility_rejects_legacy_record_with_different_light_mode() { + let dir = tempfile::tempdir().unwrap(); + let db = DB::open_test(dir.path()); + + let legacy_bytes = bincode::serialize_little(&(DB_VERSION, true)).unwrap(); + db.put(b"V", &legacy_bytes); + + let config = make_test_config(false, false, false); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + db.verify_compatibility(&config); + })); + assert!( + result.is_err(), + "verify_compatibility should reject a legacy DB with a different light_mode" + ); + } } From f5587f8e5a8991ead908aa70bf4653c6e309fbb2 Mon Sep 17 00:00:00 2001 From: Oleksandr Fedorenko Date: Mon, 14 Sep 2026 05:27:34 +0300 Subject: [PATCH 11/28] bound scripthash history scan cost and evaluate utxos_limit against the final UTXO set --- src/config.rs | 8 ++++ src/electrum/server.rs | 3 +- src/errors.rs | 5 +++ src/new_index/mempool.rs | 22 +++++++---- src/new_index/precache.rs | 2 +- src/new_index/query.rs | 16 +++++--- src/new_index/schema.rs | 81 ++++++++++++++++++++++++++++++--------- src/rest.rs | 2 +- tests/common.rs | 1 + 9 files changed, 106 insertions(+), 34 deletions(-) diff --git a/src/config.rs b/src/config.rs index 409d1c09c..944462fbd 100644 --- a/src/config.rs +++ b/src/config.rs @@ -72,6 +72,7 @@ pub struct Config { pub cors: Option, pub precache_scripts: Option, pub utxos_limit: usize, + pub history_scan_limit: usize, pub electrum_txs_limit: usize, pub electrum_subscription_limit: usize, pub electrum_checkpoint_proof_concurrency_limit: usize, @@ -282,6 +283,12 @@ impl Config { .help("Maximum number of utxos to process per address. Lookups for addresses with more utxos will fail. Applies to the Electrum and HTTP APIs.") .default_value("500") ) + .arg( + Arg::with_name("history_scan_limit") + .long("history-scan-limit") + .help("Maximum number of scripthash history rows scanned per utxo/stats lookup before giving up with a \"too popular\" error. Unlike --utxos-limit and --electrum-txs-limit, which cap the size of the result, this bounds the amount of work done per request regardless of the outcome.") + .default_value("100000") + ) .arg( Arg::with_name("electrum_txs_limit") .long("electrum-txs-limit") @@ -591,6 +598,7 @@ impl Config { daemon_conn_max_age, cookie, utxos_limit: value_t_or_exit!(m, "utxos_limit", usize), + history_scan_limit: value_t_or_exit!(m, "history_scan_limit", usize), electrum_rpc_addr, electrum_rpc_conn_max_age, electrum_rpc_max_request_num_bytes, diff --git a/src/electrum/server.rs b/src/electrum/server.rs index 44d2420ed..aa1dc3377 100644 --- a/src/electrum/server.rs +++ b/src/electrum/server.rs @@ -108,6 +108,7 @@ fn jsonrpc_code(e: &Error) -> JsonRpcV2Error { ErrorKind::InvalidParams(_) => JsonRpcV2Error::InvalidParams, ErrorKind::TooPopular | ErrorKind::TooManyUtxos + | ErrorKind::TooBigHistory | ErrorKind::TooManySubscriptions(_) => JsonRpcV2Error::BadRequest, // The daemon could not be reached (or we refused to queue for it) for a request // made on the client's behalf. This is a downstream failure, not a client error. @@ -415,7 +416,7 @@ impl Connection { #[cfg(not(feature = "liquid"))] fn blockchain_scripthash_get_balance(&self, params: &[Value]) -> Result { let script_hash = hash_from_value(params.get(0))?; - let (chain_stats, mempool_stats) = self.query.stats(&script_hash[..]); + let (chain_stats, mempool_stats) = self.query.stats(&script_hash[..])?; Ok(json!({ "confirmed": chain_stats.funded_txo_sum - chain_stats.spent_txo_sum, diff --git a/src/errors.rs b/src/errors.rs index 10a1a4d30..8c5acec9f 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -33,6 +33,11 @@ error_chain! { display("Too many unspent outputs") } + TooBigHistory { + description("Scripthash history too large to scan") + display("Scripthash history too large to scan") + } + TooManySubscriptions(limit: usize) { description("Too many subscriptions") display("Too many subscriptions on this connection (limit: {})", limit) diff --git a/src/new_index/mempool.rs b/src/new_index/mempool.rs index 08a7d2379..aea071403 100644 --- a/src/new_index/mempool.rs +++ b/src/new_index/mempool.rs @@ -179,14 +179,18 @@ impl Mempool { } #[trace] - pub fn utxo(&self, scripthash: &[u8]) -> Vec { + pub fn utxo(&self, scripthash: &[u8]) -> Result> { let _timer = self.latency.with_label_values(&["utxo"]).start_timer(); let entries = match self.history.get(scripthash) { - None => return vec![], + None => return Ok(vec![]), Some(entries) => entries, }; + ensure!( + entries.len() <= self.config.utxos_limit, + ErrorKind::TooManyUtxos + ); - entries + Ok(entries .iter() .filter_map(|entry| match entry { TxHistoryInfo::Funding(info) => { @@ -218,20 +222,24 @@ impl Mempool { | TxHistoryInfo::Pegout(_) => unreachable!(), }) .filter(|utxo| !self.has_spend(&OutPoint::from(utxo))) - .collect() + .collect()) } #[trace] // @XXX avoid code duplication with ChainQuery::stats()? - pub fn stats(&self, scripthash: &[u8]) -> ScriptStats { + pub fn stats(&self, scripthash: &[u8]) -> Result { let _timer = self.latency.with_label_values(&["stats"]).start_timer(); let mut stats = ScriptStats::default(); let mut seen_txids = HashSet::new(); let entries = match self.history.get(scripthash) { - None => return stats, + None => return Ok(stats), Some(entries) => entries, }; + ensure!( + entries.len() <= self.config.utxos_limit, + ErrorKind::TooBigHistory + ); for entry in entries { if seen_txids.insert(entry.get_txid()) { @@ -268,7 +276,7 @@ impl Mempool { }; } - stats + Ok(stats) } #[trace] diff --git a/src/new_index/precache.rs b/src/new_index/precache.rs index 86c0c6bf2..86bd6a9bc 100644 --- a/src/new_index/precache.rs +++ b/src/new_index/precache.rs @@ -32,7 +32,7 @@ pub fn precache(chain: &ChainQuery, scripthashes: Vec) { if i % 5 == 0 { info!("running pre-cache for scripthash {}/{}", i + 1, total); } - chain.stats(&scripthash[..]); + let _ = chain.stats(&scripthash[..]); //chain.utxo(&scripthash[..]); }) }); diff --git a/src/new_index/query.rs b/src/new_index/query.rs index 9b1f8c8ca..c517c4801 100644 --- a/src/new_index/query.rs +++ b/src/new_index/query.rs @@ -125,7 +125,11 @@ impl Query { let mut utxos = self.chain.utxo(scripthash, self.config.utxos_limit)?; let mempool = self.mempool(); utxos.retain(|utxo| !mempool.has_spend(&OutPoint::from(utxo))); - utxos.extend(mempool.utxo(scripthash)); + utxos.extend(mempool.utxo(scripthash)?); + ensure!( + utxos.len() <= self.config.utxos_limit, + ErrorKind::TooManyUtxos + ); Ok(utxos) } @@ -144,11 +148,11 @@ impl Query { confirmed_txids.chain(mempool_txids).collect() } - pub fn stats(&self, scripthash: &[u8]) -> (ScriptStats, ScriptStats) { - ( - self.chain.stats(scripthash), - self.mempool().stats(scripthash), - ) + pub fn stats(&self, scripthash: &[u8]) -> Result<(ScriptStats, ScriptStats)> { + Ok(( + self.chain.stats(scripthash)?, + self.mempool().stats(scripthash)?, + )) } #[trace] diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index 95b1224ba..6d52554da 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -292,6 +292,7 @@ pub struct ChainQuery { light_mode: bool, duration: HistogramVec, network: Network, + history_scan_limit: usize, } // TODO: &[Block] should be an iterator / a queue. @@ -634,6 +635,7 @@ impl ChainQuery { daemon, light_mode: config.light_mode, network: config.network_type, + history_scan_limit: config.history_scan_limit, duration: metrics.histogram_vec( HistogramOpts::new("query_duration", "Index query duration (in seconds)"), &["name"], @@ -884,12 +886,12 @@ impl ChainQuery { let had_cache = cache.is_some(); // update utxo set with new transactions since - let (newutxos, lastblock, processed_items) = cache.map_or_else( - || self.utxo_delta(scripthash, HashMap::new(), 0, limit), - |(oldutxos, blockheight)| self.utxo_delta(scripthash, oldutxos, blockheight + 1, limit), - )?; + let (newutxos, lastblock, processed_items, hit_scan_limit) = cache.map_or_else( + || self.utxo_delta(scripthash, HashMap::new(), 0), + |(oldutxos, blockheight)| self.utxo_delta(scripthash, oldutxos, blockheight + 1), + ); - // save updated utxo set to cache + // save updated utxo set to cache, even if we bail out below if let Some(lastblock) = lastblock { if had_cache || processed_items > MIN_HISTORY_ITEMS_TO_CACHE { self.store.cache_db.write_rows( @@ -899,6 +901,15 @@ impl ChainQuery { } } + if hit_scan_limit { + bail!(ErrorKind::TooBigHistory) + } + + // check against the final resolved set, not against any point in time during the replay + if newutxos.len() > limit { + bail!(ErrorKind::TooManyUtxos) + } + // format as Utxo objects Ok(newutxos .into_iter() @@ -931,8 +942,7 @@ impl ChainQuery { scripthash: &[u8], init_utxos: UtxoMap, start_height: usize, - limit: usize, - ) -> Result<(UtxoMap, Option, usize)> { + ) -> (UtxoMap, Option, usize, bool) { let _timer = self.start_timer("utxo_delta"); let headers = self.store.indexed_headers.read().unwrap(); let history_iter = self @@ -947,9 +957,22 @@ impl ChainQuery { let mut utxos = init_utxos; let mut processed_items = 0; let mut lastblock = None; + // the last height we're sure is fully scanned: all of its rows for this scripthash + // are guaranteed to be contiguous (rows are ordered by height), so it's only safe + // once we've seen a row belonging to the height that follows it + let mut safe_lastblock = None; + let mut hit_scan_limit = false; for (history, blockid) in history_iter { + if processed_items >= self.history_scan_limit { + hit_scan_limit = true; + break; + } processed_items += 1; + + if lastblock.is_some_and(|hash| hash != blockid.hash) { + safe_lastblock = lastblock; + } lastblock = Some(blockid.hash); match history.key.txinfo { @@ -963,17 +986,18 @@ impl ChainQuery { | TxHistoryInfo::Pegin(_) | TxHistoryInfo::Pegout(_) => unreachable!(), }; + } - // abort if the utxo set size excedees the limit at any point in time - if utxos.len() > limit { - bail!(ErrorKind::TooManyUtxos) - } + // the scan ran to completion (not cut off by the cap): the last height seen is + // fully in too, since there's nothing left in the iterator for this scripthash + if !hit_scan_limit { + safe_lastblock = lastblock; } - Ok((utxos, lastblock, processed_items)) + (utxos, safe_lastblock, processed_items, hit_scan_limit) } - pub fn stats(&self, scripthash: &[u8]) -> ScriptStats { + pub fn stats(&self, scripthash: &[u8]) -> Result { let _timer = self.start_timer("stats"); // get the last known stats and the blockhash they are updated for. @@ -989,12 +1013,12 @@ impl ChainQuery { }); // update stats with new transactions since - let (newstats, lastblock) = cache.map_or_else( + let (newstats, lastblock, hit_scan_limit) = cache.map_or_else( || self.stats_delta(scripthash, ScriptStats::default(), 0), |(oldstats, blockheight)| self.stats_delta(scripthash, oldstats, blockheight + 1), ); - // save updated stats to cache + // save updated stats to cache, even if we bail out below if let Some(lastblock) = lastblock { if newstats.funded_txo_count + newstats.spent_txo_count > MIN_HISTORY_ITEMS_TO_CACHE { self.store.cache_db.write_rows( @@ -1004,7 +1028,11 @@ impl ChainQuery { } } - newstats + if hit_scan_limit { + bail!(ErrorKind::TooBigHistory) + } + + Ok(newstats) } fn stats_delta( @@ -1012,7 +1040,7 @@ impl ChainQuery { scripthash: &[u8], init_stats: ScriptStats, start_height: usize, - ) -> (ScriptStats, Option) { + ) -> (ScriptStats, Option, bool) { let _timer = self.start_timer("stats_delta"); // TODO: measure also the number of txns processed. let headers = self.store.indexed_headers.read().unwrap(); let history_iter = self @@ -1027,9 +1055,20 @@ impl ChainQuery { let mut stats = init_stats; let mut seen_txids = HashSet::new(); let mut lastblock = None; + // the last height we're sure is fully scanned; see utxo_delta for why + let mut safe_lastblock = None; + let mut processed_items = 0; + let mut hit_scan_limit = false; for (history, blockid) in history_iter { + if processed_items >= self.history_scan_limit { + hit_scan_limit = true; + break; + } + processed_items += 1; + if lastblock != Some(blockid.hash) { + safe_lastblock = lastblock; seen_txids.clear(); } @@ -1070,7 +1109,13 @@ impl ChainQuery { lastblock = Some(blockid.hash); } - (stats, lastblock) + // the scan ran to completion (not cut off by the cap): the last height seen is + // fully in too, since there's nothing left in the iterator for this scripthash + if !hit_scan_limit { + safe_lastblock = lastblock; + } + + (stats, safe_lastblock, hit_scan_limit) } pub fn address_search(&self, prefix: &str, limit: usize) -> Vec { diff --git a/src/rest.rs b/src/rest.rs index 18023f358..a1b272dff 100644 --- a/src/rest.rs +++ b/src/rest.rs @@ -1014,7 +1014,7 @@ fn handle_blocking_request( (&Method::GET, Some(script_type @ &"address"), Some(script_str), None, None, None) | (&Method::GET, Some(script_type @ &"scripthash"), Some(script_str), None, None, None) => { let script_hash = to_scripthash(script_type, script_str, config.network_type)?; - let stats = query.stats(&script_hash[..]); + let stats = query.stats(&script_hash[..])?; json_response( json!({ *script_type: script_str, diff --git a/tests/common.rs b/tests/common.rs index af21d0f3b..5bb47f021 100644 --- a/tests/common.rs +++ b/tests/common.rs @@ -114,6 +114,7 @@ impl TestRunner { cors: None, precache_scripts: None, utxos_limit: 100, + history_scan_limit: 100_000, electrum_txs_limit: 100, electrum_subscription_limit: 10_000, electrum_checkpoint_proof_concurrency_limit: 2, From 91840bb12a942649f645008a4b1e691e2038b955 Mon Sep 17 00:00:00 2001 From: Oleksandr Fedorenko Date: Mon, 14 Sep 2026 05:27:34 +0300 Subject: [PATCH 12/28] fix stats_delta double-counting when a capped scan resumes from checkpoint --- src/new_index/schema.rs | 57 ++++++++++++++++++++++++++++++++++------- 1 file changed, 48 insertions(+), 9 deletions(-) diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index 6d52554da..e3f4fc5b7 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -226,7 +226,7 @@ pub struct SpendingInput { pub confirmed: Option, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct ScriptStats { pub tx_count: usize, pub funded_txo_count: usize, @@ -795,6 +795,19 @@ impl ChainQuery { ) } + // like history_iter_scan_reverse, but seeks directly to `height` instead of the tip + fn history_iter_scan_reverse_from( + &self, + code: u8, + hash: &[u8], + height: usize, + ) -> ReverseScanIterator<'_> { + self.store.history_db.iter_scan_reverse( + &TxHistoryRow::filter(code, &hash[..]), + &TxHistoryRow::prefix_height(code, &hash[..], height as u32 + 1), + ) + } + pub fn history( &self, scripthash: &[u8], @@ -814,13 +827,33 @@ impl ChainQuery { ) -> Result> { let _timer_scan = self.start_timer("history"); let headers = self.store.indexed_headers.read().unwrap(); - let history_iter = self - .history_iter_scan_reverse(code, hash) + + // a cursor can only match a confirmed tx; resolve its height up front, or bail + let seek_height = match last_seen_txid { + Some(txid) => match self.tx_confirming_block(txid) { + Some(blockid) => Some(blockid.height), + None => return vec![], + }, + None => None, + }; + + let raw_iter = match seek_height { + Some(height) => self.history_iter_scan_reverse_from(code, hash, height), + None => self.history_iter_scan_reverse(code, hash), + }; + + let mut scanned = 0usize; + let scan_limit = self.history_scan_limit; + let history_iter = raw_iter + // bounds skip_while below, in case the cursor belongs to another scripthash + .take_while(move |_| { + scanned += 1; + scanned <= scan_limit + }) .map(TxHistoryRow::from_row) .map(|row| (row.get_txid(), row.key.confirmed_height as usize)) // XXX: unique() requires keeping an in-memory list of all txids, can we avoid that? .unique() - // TODO seek directly to last seen tx without reading earlier rows .skip_while(|(txid, _)| { // skip until we reach the last_seen_txid last_seen_txid.map_or(false, |last_seen_txid| last_seen_txid != txid) @@ -1052,10 +1085,12 @@ impl ChainQuery { Some((history, BlockId::from(header))) }); - let mut stats = init_stats; + let mut stats = init_stats.clone(); + // stats as of the last fully-scanned height; unlike UtxoMap these counters + // aren't idempotent, so a capped scan rolls back to this instead of the pointer + let mut committed_stats = init_stats; let mut seen_txids = HashSet::new(); let mut lastblock = None; - // the last height we're sure is fully scanned; see utxo_delta for why let mut safe_lastblock = None; let mut processed_items = 0; let mut hit_scan_limit = false; @@ -1068,6 +1103,7 @@ impl ChainQuery { processed_items += 1; if lastblock != Some(blockid.hash) { + committed_stats = stats.clone(); safe_lastblock = lastblock; seen_txids.clear(); } @@ -1109,9 +1145,12 @@ impl ChainQuery { lastblock = Some(blockid.hash); } - // the scan ran to completion (not cut off by the cap): the last height seen is - // fully in too, since there's nothing left in the iterator for this scripthash - if !hit_scan_limit { + if hit_scan_limit { + // discard the not-yet-committed (possibly partial) tail height + stats = committed_stats; + } else { + // the scan ran to completion: the last height seen is fully in too, since + // there's nothing left in the iterator for this scripthash safe_lastblock = lastblock; } From 9a7f3042c5f271424871baa627a380cc3f3a270e Mon Sep 17 00:00:00 2001 From: Oleksandr Fedorenko Date: Mon, 14 Sep 2026 05:27:34 +0300 Subject: [PATCH 13/28] add integration tests for utxo/history scan limits --- tests/common.rs | 109 +++++++++++++++++++++++++++++- tests/scan_limits.rs | 156 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 262 insertions(+), 3 deletions(-) create mode 100644 tests/scan_limits.rs diff --git a/tests/common.rs b/tests/common.rs index 5bb47f021..546fcf874 100644 --- a/tests/common.rs +++ b/tests/common.rs @@ -45,6 +45,10 @@ pub struct TestRunner { impl TestRunner { pub fn new() -> Result { + Self::new_with(|_| {}) + } + + pub fn new_with(configure: impl FnOnce(&mut Config)) -> Result { let log = init_log(); // Setup the bitcoind/elementsd config @@ -89,7 +93,7 @@ impl TestRunner { let electrsdb = tempfile::tempdir().unwrap(); - let config = Arc::new(Config { + let mut config = Config { log, network_type, db_path: electrsdb.path().to_path_buf(), @@ -137,7 +141,9 @@ impl TestRunner { //electrum_announce: bool, //#[cfg(feature = "electrum-discovery")] //tor_proxy: Option, - }); + }; + configure(&mut config); + let config = Arc::new(config); let signal = Waiter::start(crossbeam_channel::never()); let metrics_addr = rand_available_addr(); @@ -264,6 +270,98 @@ impl TestRunner { Ok(txid) } + /// Send `count` outputs of `amount` each to `addr` within a single transaction. + // createrawtransaction rejects duplicate-address outputs, so this is built by hand. + #[cfg(not(feature = "liquid"))] + pub fn send_multi( + &mut self, + addr: &Address, + amount: bitcoin::Amount, + count: usize, + ) -> Result { + let fee = bitcoin::Amount::from_sat(50_000); + let total_out = bitcoin::Amount::from_sat(amount.to_sat() * count as u64); + + let utxos: Vec = self.node_client().call("listunspent", &[])?; + let utxo = utxos + .iter() + .max_by(|a, b| { + a["amount"] + .as_f64() + .unwrap() + .total_cmp(&b["amount"].as_f64().unwrap()) + }) + .expect("wallet has no spendable utxos"); + let input_value = bitcoin::Amount::from_btc(utxo["amount"].as_f64().unwrap()).unwrap(); + let change = input_value - total_out - fee; + + let input = bitcoin::TxIn { + previous_output: bitcoin::OutPoint { + txid: utxo["txid"].as_str().unwrap().parse().unwrap(), + vout: utxo["vout"].as_u64().unwrap() as u32, + }, + ..Default::default() + }; + let mut outputs: Vec = (0..count) + .map(|_| bitcoin::TxOut { + value: amount, + script_pubkey: addr.script_pubkey(), + }) + .collect(); + outputs.push(bitcoin::TxOut { + value: change, + script_pubkey: self.newaddress()?.script_pubkey(), + }); + + let tx = bitcoin::Transaction { + version: bitcoin::transaction::Version::TWO, + lock_time: bitcoin::absolute::LockTime::ZERO, + input: vec![input], + output: outputs, + }; + let raw_hex = bitcoin::consensus::encode::serialize_hex(&tx); + let signed: serde_json::Value = self + .node_client() + .call("signrawtransactionwithwallet", &[json!(raw_hex)])?; + let signed_hex = signed["hex"].as_str().unwrap().to_string(); + let txid: Txid = self + .node_client() + .call("sendrawtransaction", &[json!(signed_hex)])?; + self.sync()?; + Ok(txid) + } + + /// Spend outputs `0..n_outputs` of `txid` (assumed wallet-owned) into a single output + /// paid to `to`. + #[cfg(not(feature = "liquid"))] + pub fn consolidate( + &mut self, + txid: Txid, + n_outputs: usize, + amount_per_output: bitcoin::Amount, + fee: bitcoin::Amount, + to: &Address, + ) -> Result { + let inputs: Vec = (0..n_outputs as u32) + .map(|vout| json!({ "txid": txid.to_string(), "vout": vout })) + .collect(); + let total = bitcoin::Amount::from_sat(amount_per_output.to_sat() * n_outputs as u64); + let output_amount = bitcoin::Amount::from_sat(total.to_sat() - fee.to_sat()); + let outputs = json!({ to.to_string(): output_amount.to_btc() }); + let raw_hex: String = self + .node_client() + .call("createrawtransaction", &[json!(inputs), outputs])?; + let signed: serde_json::Value = self + .node_client() + .call("signrawtransactionwithwallet", &[json!(raw_hex)])?; + let signed_hex = signed["hex"].as_str().unwrap().to_string(); + let txid: Txid = self + .node_client() + .call("sendrawtransaction", &[json!(signed_hex)])?; + self.sync()?; + Ok(txid) + } + #[cfg(feature = "liquid")] pub fn send_asset( &mut self, @@ -350,7 +448,12 @@ impl TestRunner { } pub fn init_rest_tester() -> Result<(rest::Handle, net::SocketAddr, TestRunner)> { - let tester = TestRunner::new()?; + init_rest_tester_with(|_| {}) +} +pub fn init_rest_tester_with( + configure: impl FnOnce(&mut Config), +) -> Result<(rest::Handle, net::SocketAddr, TestRunner)> { + let tester = TestRunner::new_with(configure)?; let addr = tester.config.http_addr; let rest_server = rest::start(Arc::clone(&tester.config), Arc::clone(&tester.query)); wait_for_tcp(addr, "REST"); diff --git a/tests/scan_limits.rs b/tests/scan_limits.rs new file mode 100644 index 000000000..c21ec9dca --- /dev/null +++ b/tests/scan_limits.rs @@ -0,0 +1,156 @@ +use serde_json::Value; +use std::net; + +use electrs::chain::Txid; + +pub mod common; + +use common::Result; + +fn get_json(rest_addr: net::SocketAddr, path: &str) -> Result { + Ok(ureq::get(&format!("http://{}{}", rest_addr, path)) + .call()? + .into_body() + .read_json()?) +} + +// like get_json, but doesn't treat a non-2xx response as an error +fn get_allow_error( + rest_addr: net::SocketAddr, + path: &str, +) -> std::result::Result, ureq::Error> { + ureq::get(&format!("http://{}{}", rest_addr, path)) + .config() + .http_status_as_error(false) + .build() + .call() +} + +// utxos_limit must apply to the final UTXO set, not any point during the replay. +#[cfg(not(feature = "liquid"))] +#[test] +fn test_utxo_limit_checked_against_final_state() -> Result<()> { + let (rest_handle, rest_addr, mut tester) = common::init_rest_tester().unwrap(); + + let addr1 = tester.newaddress()?; + let amount = bitcoin::Amount::from_sat(5_000); + + // 101 outputs in one tx -- one more than utxos_limit (100, tests/common.rs) + let fund_txid = tester.send_multi(&addr1, amount, 101)?; + tester.mine()?; + + let resp = get_allow_error(rest_addr, &format!("/address/{}/utxo", addr1))?; + assert_eq!(resp.status(), 400); + assert_eq!( + resp.into_body().read_to_string()?, + "Too many unspent outputs" + ); + + let stats = get_json(rest_addr, &format!("/address/{}", addr1))?; + assert_eq!(stats["chain_stats"]["funded_txo_count"].as_u64(), Some(101)); + + tester.consolidate( + fund_txid, + 101, + amount, + bitcoin::Amount::from_sat(30_000), + &addr1, + )?; + tester.mine()?; + + let res = get_json(rest_addr, &format!("/address/{}/utxo", addr1))?; + let utxos = res.as_array().expect("array of utxos"); + assert_eq!(utxos.len(), 1); + + rest_handle.stop(); + Ok(()) +} + +// history_scan_limit must bound the scan, and resume across repeated requests. +#[test] +fn test_history_scan_limit_bounds_and_resumes() -> Result<()> { + // above MIN_HISTORY_ITEMS_TO_CACHE (100), so the first capped scan actually gets cached + let (rest_handle, rest_addr, mut tester) = + common::init_rest_tester_with(|c| c.history_scan_limit = 110).unwrap(); + + let addr1 = tester.newaddress()?; + let amount = bitcoin::Amount::from_sat(1_000); + + // 130 confirmed sends, one output per block -- more than history_scan_limit + for _ in 0..130 { + tester.send(&addr1, amount)?; + tester.mine()?; + } + + // first request: only the first ~110 rows fit under the cap + let resp = get_allow_error(rest_addr, &format!("/address/{}", addr1))?; + assert_eq!(resp.status(), 400); + assert_eq!( + resp.into_body().read_to_string()?, + "Scripthash history too large to scan" + ); + + // a handful of retries should be enough to resume from the checkpoint and catch up + let mut stats = None; + for _ in 0..5 { + match get_json(rest_addr, &format!("/address/{}", addr1)) { + Ok(res) => { + stats = Some(res); + break; + } + Err(_) => continue, + } + } + let stats = stats.expect("scan should have converged within a few requests"); + assert_eq!(stats["chain_stats"]["funded_txo_count"].as_u64(), Some(130)); + + rest_handle.stop(); + Ok(()) +} + +// an unmatched last_seen_txid cursor must return an empty page, not a full scan. +#[test] +fn test_history_cursor_unmatched_returns_empty() -> Result<()> { + let (rest_handle, rest_addr, mut tester) = common::init_rest_tester().unwrap(); + + // fund addr2 before addr1 has any UTXOs, so the wallet can't pick addr1 as an input + let addr2 = tester.newaddress()?; + let foreign_txid = tester.send(&addr2, bitcoin::Amount::from_sat(100_000))?; + tester.mine()?; + + let addr1 = tester.newaddress()?; + let mut addr1_txids: Vec = vec![]; + for _ in 0..3 { + addr1_txids.push(tester.send(&addr1, bitcoin::Amount::from_sat(100_000))?); + tester.mine()?; + } + + // sanity check: a real cursor works and excludes the cursor tx itself + let cursor = addr1_txids[1]; + let res = get_json( + rest_addr, + &format!("/address/{}/txs/chain/{}", addr1, cursor), + )?; + let txs = res.as_array().expect("array of txs"); + assert!(txs + .iter() + .all(|tx| tx["txid"].as_str() != Some(cursor.to_string().as_str()))); + + // a confirmed txid that belongs to a different scripthash's history + let res = get_json( + rest_addr, + &format!("/address/{}/txs/chain/{}", addr1, foreign_txid), + )?; + assert_eq!(res.as_array().expect("array of txs").len(), 0); + + // a syntactically valid txid that was never confirmed at all + let never_confirmed = "22".repeat(32); + let res = get_json( + rest_addr, + &format!("/address/{}/txs/chain/{}", addr1, never_confirmed), + )?; + assert_eq!(res.as_array().expect("array of txs").len(), 0); + + rest_handle.stop(); + Ok(()) +} From 008598698948b6e35efa365537faabc25d1977c8 Mon Sep 17 00:00:00 2001 From: Oleksandr Fedorenko Date: Mon, 14 Sep 2026 05:27:34 +0300 Subject: [PATCH 14/28] only cap history scans at height boundaries to avoid stalling forever --- src/new_index/schema.rs | 54 +++++++++++++++-------------------------- tests/common.rs | 4 --- tests/scan_limits.rs | 47 +++++++++++++++++++++++++---------- 3 files changed, 53 insertions(+), 52 deletions(-) diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index e3f4fc5b7..c66f39327 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -226,7 +226,7 @@ pub struct SpendingInput { pub confirmed: Option, } -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug)] pub struct ScriptStats { pub tx_count: usize, pub funded_txo_count: usize, @@ -795,7 +795,6 @@ impl ChainQuery { ) } - // like history_iter_scan_reverse, but seeks directly to `height` instead of the tip fn history_iter_scan_reverse_from( &self, code: u8, @@ -828,9 +827,8 @@ impl ChainQuery { let _timer_scan = self.start_timer("history"); let headers = self.store.indexed_headers.read().unwrap(); - // a cursor can only match a confirmed tx; resolve its height up front, or bail let seek_height = match last_seen_txid { - Some(txid) => match self.tx_confirming_block(txid) { + Some(txid) => match self.tx_confirming_block_with_headers(txid, &headers) { Some(blockid) => Some(blockid.height), None => return vec![], }, @@ -845,7 +843,6 @@ impl ChainQuery { let mut scanned = 0usize; let scan_limit = self.history_scan_limit; let history_iter = raw_iter - // bounds skip_while below, in case the cursor belongs to another scripthash .take_while(move |_| { scanned += 1; scanned <= scan_limit @@ -938,7 +935,6 @@ impl ChainQuery { bail!(ErrorKind::TooBigHistory) } - // check against the final resolved set, not against any point in time during the replay if newutxos.len() > limit { bail!(ErrorKind::TooManyUtxos) } @@ -990,22 +986,18 @@ impl ChainQuery { let mut utxos = init_utxos; let mut processed_items = 0; let mut lastblock = None; - // the last height we're sure is fully scanned: all of its rows for this scripthash - // are guaranteed to be contiguous (rows are ordered by height), so it's only safe - // once we've seen a row belonging to the height that follows it let mut safe_lastblock = None; let mut hit_scan_limit = false; for (history, blockid) in history_iter { - if processed_items >= self.history_scan_limit { - hit_scan_limit = true; - break; - } - processed_items += 1; - if lastblock.is_some_and(|hash| hash != blockid.hash) { safe_lastblock = lastblock; + if processed_items >= self.history_scan_limit { + hit_scan_limit = true; + break; + } } + processed_items += 1; lastblock = Some(blockid.hash); match history.key.txinfo { @@ -1021,8 +1013,6 @@ impl ChainQuery { }; } - // the scan ran to completion (not cut off by the cap): the last height seen is - // fully in too, since there's nothing left in the iterator for this scripthash if !hit_scan_limit { safe_lastblock = lastblock; } @@ -1085,10 +1075,7 @@ impl ChainQuery { Some((history, BlockId::from(header))) }); - let mut stats = init_stats.clone(); - // stats as of the last fully-scanned height; unlike UtxoMap these counters - // aren't idempotent, so a capped scan rolls back to this instead of the pointer - let mut committed_stats = init_stats; + let mut stats = init_stats; let mut seen_txids = HashSet::new(); let mut lastblock = None; let mut safe_lastblock = None; @@ -1096,17 +1083,15 @@ impl ChainQuery { let mut hit_scan_limit = false; for (history, blockid) in history_iter { - if processed_items >= self.history_scan_limit { - hit_scan_limit = true; - break; - } - processed_items += 1; - if lastblock != Some(blockid.hash) { - committed_stats = stats.clone(); safe_lastblock = lastblock; seen_txids.clear(); + if processed_items >= self.history_scan_limit { + hit_scan_limit = true; + break; + } } + processed_items += 1; if seen_txids.insert(history.get_txid()) { stats.tx_count += 1; @@ -1145,12 +1130,7 @@ impl ChainQuery { lastblock = Some(blockid.hash); } - if hit_scan_limit { - // discard the not-yet-committed (possibly partial) tail height - stats = committed_stats; - } else { - // the scan ran to completion: the last height seen is fully in too, since - // there's nothing left in the iterator for this scripthash + if !hit_scan_limit { safe_lastblock = lastblock; } @@ -1370,9 +1350,13 @@ impl ChainQuery { pub fn tx_confirming_block(&self, txid: &Txid) -> Option { let _timer = self.start_timer("tx_confirming_block"); + let headers = self.store.indexed_headers.read().unwrap(); + self.tx_confirming_block_with_headers(txid, &headers) + } + + fn tx_confirming_block_with_headers(&self, txid: &Txid, headers: &HeaderList) -> Option { let row_value = self.store.history_db.get(&TxConfRow::key(txid))?; let height = TxConfRow::height_from_val(&row_value); - let headers = self.store.indexed_headers.read().unwrap(); // skip over entries that point to non-existing heights (may happen while new/reorged blocks are being processed) Some(headers.header_by_height(height as usize)?.into()) } diff --git a/tests/common.rs b/tests/common.rs index 546fcf874..04d198f01 100644 --- a/tests/common.rs +++ b/tests/common.rs @@ -270,8 +270,6 @@ impl TestRunner { Ok(txid) } - /// Send `count` outputs of `amount` each to `addr` within a single transaction. - // createrawtransaction rejects duplicate-address outputs, so this is built by hand. #[cfg(not(feature = "liquid"))] pub fn send_multi( &mut self, @@ -331,8 +329,6 @@ impl TestRunner { Ok(txid) } - /// Spend outputs `0..n_outputs` of `txid` (assumed wallet-owned) into a single output - /// paid to `to`. #[cfg(not(feature = "liquid"))] pub fn consolidate( &mut self, diff --git a/tests/scan_limits.rs b/tests/scan_limits.rs index c21ec9dca..09e3417f7 100644 --- a/tests/scan_limits.rs +++ b/tests/scan_limits.rs @@ -14,7 +14,6 @@ fn get_json(rest_addr: net::SocketAddr, path: &str) -> Result { .read_json()?) } -// like get_json, but doesn't treat a non-2xx response as an error fn get_allow_error( rest_addr: net::SocketAddr, path: &str, @@ -26,7 +25,6 @@ fn get_allow_error( .call() } -// utxos_limit must apply to the final UTXO set, not any point during the replay. #[cfg(not(feature = "liquid"))] #[test] fn test_utxo_limit_checked_against_final_state() -> Result<()> { @@ -35,7 +33,6 @@ fn test_utxo_limit_checked_against_final_state() -> Result<()> { let addr1 = tester.newaddress()?; let amount = bitcoin::Amount::from_sat(5_000); - // 101 outputs in one tx -- one more than utxos_limit (100, tests/common.rs) let fund_txid = tester.send_multi(&addr1, amount, 101)?; tester.mine()?; @@ -66,23 +63,53 @@ fn test_utxo_limit_checked_against_final_state() -> Result<()> { Ok(()) } -// history_scan_limit must bound the scan, and resume across repeated requests. +#[cfg(not(feature = "liquid"))] +#[test] +fn test_history_scan_limit_finishes_oversized_height() -> Result<()> { + let (rest_handle, rest_addr, mut tester) = + common::init_rest_tester_with(|c| c.history_scan_limit = 5).unwrap(); + + let addr1 = tester.newaddress()?; + let amount = bitcoin::Amount::from_sat(5_000); + + tester.send_multi(&addr1, amount, 150)?; + tester.mine()?; + + for _ in 0..3 { + tester.send(&addr1, amount)?; + tester.mine()?; + } + + let mut stats = None; + for _ in 0..5 { + match get_json(rest_addr, &format!("/address/{}", addr1)) { + Ok(res) => { + stats = Some(res); + break; + } + Err(_) => continue, + } + } + let stats = stats.expect("scan should get past the oversized height and converge"); + assert_eq!(stats["chain_stats"]["funded_txo_count"].as_u64(), Some(153)); + + rest_handle.stop(); + Ok(()) +} + #[test] fn test_history_scan_limit_bounds_and_resumes() -> Result<()> { - // above MIN_HISTORY_ITEMS_TO_CACHE (100), so the first capped scan actually gets cached let (rest_handle, rest_addr, mut tester) = common::init_rest_tester_with(|c| c.history_scan_limit = 110).unwrap(); let addr1 = tester.newaddress()?; let amount = bitcoin::Amount::from_sat(1_000); - // 130 confirmed sends, one output per block -- more than history_scan_limit for _ in 0..130 { tester.send(&addr1, amount)?; tester.mine()?; } - // first request: only the first ~110 rows fit under the cap let resp = get_allow_error(rest_addr, &format!("/address/{}", addr1))?; assert_eq!(resp.status(), 400); assert_eq!( @@ -90,7 +117,6 @@ fn test_history_scan_limit_bounds_and_resumes() -> Result<()> { "Scripthash history too large to scan" ); - // a handful of retries should be enough to resume from the checkpoint and catch up let mut stats = None; for _ in 0..5 { match get_json(rest_addr, &format!("/address/{}", addr1)) { @@ -108,12 +134,10 @@ fn test_history_scan_limit_bounds_and_resumes() -> Result<()> { Ok(()) } -// an unmatched last_seen_txid cursor must return an empty page, not a full scan. #[test] fn test_history_cursor_unmatched_returns_empty() -> Result<()> { let (rest_handle, rest_addr, mut tester) = common::init_rest_tester().unwrap(); - // fund addr2 before addr1 has any UTXOs, so the wallet can't pick addr1 as an input let addr2 = tester.newaddress()?; let foreign_txid = tester.send(&addr2, bitcoin::Amount::from_sat(100_000))?; tester.mine()?; @@ -125,7 +149,6 @@ fn test_history_cursor_unmatched_returns_empty() -> Result<()> { tester.mine()?; } - // sanity check: a real cursor works and excludes the cursor tx itself let cursor = addr1_txids[1]; let res = get_json( rest_addr, @@ -136,14 +159,12 @@ fn test_history_cursor_unmatched_returns_empty() -> Result<()> { .iter() .all(|tx| tx["txid"].as_str() != Some(cursor.to_string().as_str()))); - // a confirmed txid that belongs to a different scripthash's history let res = get_json( rest_addr, &format!("/address/{}/txs/chain/{}", addr1, foreign_txid), )?; assert_eq!(res.as_array().expect("array of txs").len(), 0); - // a syntactically valid txid that was never confirmed at all let never_confirmed = "22".repeat(32); let res = get_json( rest_addr, From 8bf81bbc1e05cc8b7b1b3ba575af5609f87309a1 Mon Sep 17 00:00:00 2001 From: Oleksandr Fedorenko Date: Tue, 15 Sep 2026 15:57:25 +0300 Subject: [PATCH 15/28] fix mempool scan bound, history pagination truncation --- src/new_index/mempool.rs | 2 +- src/new_index/schema.rs | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/new_index/mempool.rs b/src/new_index/mempool.rs index aea071403..88f73cbc1 100644 --- a/src/new_index/mempool.rs +++ b/src/new_index/mempool.rs @@ -187,7 +187,7 @@ impl Mempool { }; ensure!( entries.len() <= self.config.utxos_limit, - ErrorKind::TooManyUtxos + ErrorKind::TooBigHistory ); Ok(entries diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index c66f39327..9f881a500 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -830,7 +830,7 @@ impl ChainQuery { let seek_height = match last_seen_txid { Some(txid) => match self.tx_confirming_block_with_headers(txid, &headers) { Some(blockid) => Some(blockid.height), - None => return vec![], + None => return Ok(vec![]), }, None => None, }; @@ -841,11 +841,17 @@ impl ChainQuery { }; let mut scanned = 0usize; + let mut hit_scan_limit = false; let scan_limit = self.history_scan_limit; let history_iter = raw_iter - .take_while(move |_| { + .take_while(|_| { scanned += 1; - scanned <= scan_limit + if scanned > scan_limit { + hit_scan_limit = true; + false + } else { + true + } }) .map(TxHistoryRow::from_row) .map(|row| (row.get_txid(), row.key.confirmed_height as usize)) @@ -871,6 +877,10 @@ impl ChainQuery { } drop(headers); + if hit_scan_limit && txids_with_blockhash.is_empty() { + bail!(ErrorKind::TooBigHistory); + } + Ok(self .lookup_txns(&txids_with_blockhash)? .into_iter() From bfb0c94eb5e6712c0a9bc9e611887afd421704f9 Mon Sep 17 00:00:00 2001 From: Oleksandr Fedorenko Date: Fri, 18 Sep 2026 12:43:57 +0300 Subject: [PATCH 16/28] lock in that the utxos-limit holds until consolidation confirms --- tests/scan_limits.rs | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/scan_limits.rs b/tests/scan_limits.rs index 09e3417f7..c060c7a26 100644 --- a/tests/scan_limits.rs +++ b/tests/scan_limits.rs @@ -63,6 +63,41 @@ fn test_utxo_limit_checked_against_final_state() -> Result<()> { Ok(()) } +#[cfg(not(feature = "liquid"))] +#[test] +fn test_utxo_limit_stays_until_consolidation_confirms() -> Result<()> { + let (rest_handle, rest_addr, mut tester) = common::init_rest_tester().unwrap(); + + let addr1 = tester.newaddress()?; + let amount = bitcoin::Amount::from_sat(5_000); + + let fund_txid = tester.send_multi(&addr1, amount, 101)?; + tester.mine()?; + + let resp = get_allow_error(rest_addr, &format!("/address/{}/utxo", addr1))?; + assert_eq!(resp.status(), 400); + + tester.consolidate( + fund_txid, + 101, + amount, + bitcoin::Amount::from_sat(30_000), + &addr1, + )?; + + let resp = get_allow_error(rest_addr, &format!("/address/{}/utxo", addr1))?; + assert_eq!(resp.status(), 400); + + // Once mined, the limit is evaluated against the new confirmed state and clears. + tester.mine()?; + let res = get_json(rest_addr, &format!("/address/{}/utxo", addr1))?; + let utxos = res.as_array().expect("array of utxos"); + assert_eq!(utxos.len(), 1); + + rest_handle.stop(); + Ok(()) +} + #[cfg(not(feature = "liquid"))] #[test] fn test_history_scan_limit_finishes_oversized_height() -> Result<()> { From a233207cb51fd9680da1e1103f4867d1c50cdb79 Mon Sep 17 00:00:00 2001 From: Oleksandr Fedorenko Date: Tue, 22 Sep 2026 18:54:12 +0300 Subject: [PATCH 17/28] save a checkpoint at the scan limit and reject truncated history pages --- src/config.rs | 2 +- src/new_index/schema.rs | 8 ++-- tests/scan_limits.rs | 95 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 4 deletions(-) diff --git a/src/config.rs b/src/config.rs index 944462fbd..6eb6be6ed 100644 --- a/src/config.rs +++ b/src/config.rs @@ -286,7 +286,7 @@ impl Config { .arg( Arg::with_name("history_scan_limit") .long("history-scan-limit") - .help("Maximum number of scripthash history rows scanned per utxo/stats lookup before giving up with a \"too popular\" error. Unlike --utxos-limit and --electrum-txs-limit, which cap the size of the result, this bounds the amount of work done per request regardless of the outcome.") + .help("Approximate number of scripthash history rows scanned per utxo/stats lookup before giving up with a \"too popular\" error (the scan reads through the end of the height it is on when the limit is reached, so it may process somewhat more). Unlike --utxos-limit and --electrum-txs-limit, which cap the size of the result, this bounds the amount of work done per request regardless of the outcome.") .default_value("100000") ) .arg( diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index 9f881a500..a357848d9 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -877,7 +877,7 @@ impl ChainQuery { } drop(headers); - if hit_scan_limit && txids_with_blockhash.is_empty() { + if hit_scan_limit && txids_with_blockhash.len() < limit { bail!(ErrorKind::TooBigHistory); } @@ -933,7 +933,7 @@ impl ChainQuery { // save updated utxo set to cache, even if we bail out below if let Some(lastblock) = lastblock { - if had_cache || processed_items > MIN_HISTORY_ITEMS_TO_CACHE { + if had_cache || hit_scan_limit || processed_items > MIN_HISTORY_ITEMS_TO_CACHE { self.store.cache_db.write_rows( vec![UtxoCacheRow::new(scripthash, &newutxos, &lastblock).into_row()], DBFlush::Enable, @@ -1053,7 +1053,9 @@ impl ChainQuery { // save updated stats to cache, even if we bail out below if let Some(lastblock) = lastblock { - if newstats.funded_txo_count + newstats.spent_txo_count > MIN_HISTORY_ITEMS_TO_CACHE { + if hit_scan_limit + || newstats.funded_txo_count + newstats.spent_txo_count > MIN_HISTORY_ITEMS_TO_CACHE + { self.store.cache_db.write_rows( vec![StatsCacheRow::new(scripthash, &newstats, &lastblock).into_row()], DBFlush::Enable, diff --git a/tests/scan_limits.rs b/tests/scan_limits.rs index c060c7a26..692849826 100644 --- a/tests/scan_limits.rs +++ b/tests/scan_limits.rs @@ -169,6 +169,101 @@ fn test_history_scan_limit_bounds_and_resumes() -> Result<()> { Ok(()) } +#[test] +fn test_history_scan_limit_checkpoints_below_cache_threshold() -> Result<()> { + let (rest_handle, rest_addr, mut tester) = + common::init_rest_tester_with(|c| c.history_scan_limit = 5).unwrap(); + + let addr1 = tester.newaddress()?; + let amount = bitcoin::Amount::from_sat(1_000); + + for _ in 0..6 { + tester.send(&addr1, amount)?; + tester.mine()?; + } + + let resp = get_allow_error(rest_addr, &format!("/address/{}", addr1))?; + assert_eq!(resp.status(), 400); + + let mut stats = None; + for _ in 0..3 { + match get_json(rest_addr, &format!("/address/{}", addr1)) { + Ok(res) => { + stats = Some(res); + break; + } + Err(_) => continue, + } + } + let stats = + stats.expect("checkpoint below the cache threshold should still be saved and progress"); + assert_eq!(stats["chain_stats"]["funded_txo_count"].as_u64(), Some(6)); + + rest_handle.stop(); + Ok(()) +} + +#[cfg(not(feature = "liquid"))] +#[test] +fn test_utxo_scan_limit_checkpoints_below_cache_threshold() -> Result<()> { + let (rest_handle, rest_addr, mut tester) = + common::init_rest_tester_with(|c| c.history_scan_limit = 5).unwrap(); + + let addr1 = tester.newaddress()?; + let amount = bitcoin::Amount::from_sat(1_000); + + for _ in 0..6 { + tester.send_multi(&addr1, amount, 1)?; + tester.mine()?; + } + + let resp = get_allow_error(rest_addr, &format!("/address/{}/utxo", addr1))?; + assert_eq!(resp.status(), 400); + + let mut utxos = None; + for _ in 0..3 { + match get_json(rest_addr, &format!("/address/{}/utxo", addr1)) { + Ok(res) => { + utxos = Some(res); + break; + } + Err(_) => continue, + } + } + let utxos = + utxos.expect("checkpoint below the cache threshold should still be saved and progress"); + assert_eq!(utxos.as_array().expect("array of utxos").len(), 6); + + rest_handle.stop(); + Ok(()) +} + +#[cfg(not(feature = "liquid"))] +#[test] +fn test_history_scan_limit_errors_on_short_page() -> Result<()> { + let (rest_handle, rest_addr, mut tester) = + common::init_rest_tester_with(|c| c.history_scan_limit = 5).unwrap(); + + let addr1 = tester.newaddress()?; + let amount = bitcoin::Amount::from_sat(1_000); + + tester.send(&addr1, amount)?; + tester.mine()?; + + tester.send_multi(&addr1, amount, 20)?; + tester.mine()?; + + let resp = get_allow_error(rest_addr, &format!("/address/{}/txs/chain", addr1))?; + assert_eq!(resp.status(), 400); + assert_eq!( + resp.into_body().read_to_string()?, + "Scripthash history too large to scan" + ); + + rest_handle.stop(); + Ok(()) +} + #[test] fn test_history_cursor_unmatched_returns_empty() -> Result<()> { let (rest_handle, rest_addr, mut tester) = common::init_rest_tester().unwrap(); From f3332f4ed7f69dd48c2abfd1f574746b5fae02ec Mon Sep 17 00:00:00 2001 From: Randall Naar Date: Tue, 15 Sep 2026 12:25:12 -0400 Subject: [PATCH 18/28] Added rpc response limits. --- README.md | 3 + src/config.rs | 87 +++ src/electrum/client_io.rs | 363 ++++++++++++ src/electrum/mod.rs | 2 + src/electrum/response.rs | 934 ++++++++++++++++++++++++++++++ src/electrum/server.rs | 157 ++--- src/errors.rs | 15 + src/new_index/db.rs | 4 + tests/common.rs | 17 +- tests/electrum_response_limits.rs | 294 ++++++++++ 10 files changed, 1799 insertions(+), 77 deletions(-) create mode 100644 src/electrum/client_io.rs create mode 100644 src/electrum/response.rs create mode 100644 tests/electrum_response_limits.rs diff --git a/README.md b/README.md index 7a5f93e35..e53c71f4b 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,9 @@ In addition to electrs's original configuration options, a few new options are a - `--enable-mining-rest` - enables cached mining-related HTTP endpoints. - `--utxos-limit ` - maximum number of utxos to return per address. - `--electrum-txs-limit ` - maximum number of txs to return per address in the electrum server (does not apply for the http api). +- `--electrum-rpc-write-timeout ` - maximum time to transmit a complete Electrum reply, including a whole batch and its newline, an error reply, or a subscription notification. The deadline starts at the first write and is not extended by partial progress. Expiry closes the connection and releases its response buffers. Idle subscriptions and command execution are unaffected. Default: `30`, matching the RPC proxy's `CLIENT_WRITE_TIMEOUT_SECS`; `0` disables the bound. The `electrum_client_write_timeouts_total` counter records connections closed by this deadline. +- `--electrum-rpc-max-response-num-bytes ` - maximum size, in bytes, of a single Electrum solicited reply line (brackets, commas, error objects, and newline included). Overflowing replies return a correlated error (code 1) and the rest of the batch is not executed; the command that overflowed may already have completed. Subscription notifications are exempt. `0` disables the cap. Default: `8388608` (8 MiB) on Bitcoin, `33554432` (32 MiB) on Liquid (sized for a 2016-header window during a dynafed parameter vote). +- `--electrum-rpc-global-response-budget-bytes ` - aggregate cap, in bytes, on Electrum response-buffer memory retained across all connections at any instant. Small replies below a 16 KiB per-connection floor are exempt; above the floor each reply charges the budget in 64 KiB chunks (or a smaller final chunk) and releases on drop. Requests that would exceed the budget are rejected with a server error (code 2) instead of allocating, and the connection stays usable. Must be `>= --electrum-rpc-max-response-num-bytes`. `0` disables the budget. Default: `67108864` (64 MiB) on Bitcoin, `268435456` (256 MiB) on Liquid. - `--electrum-banner ` - welcome banner text for electrum server. Additional options with the `liquid` feature: diff --git a/src/config.rs b/src/config.rs index 6eb6be6ed..ec33366e3 100644 --- a/src/config.rs +++ b/src/config.rs @@ -60,7 +60,10 @@ pub struct Config { pub cookie: Option, pub electrum_rpc_addr: SocketAddr, pub electrum_rpc_conn_max_age: Option, + pub electrum_rpc_write_timeout: Option, pub electrum_rpc_max_request_num_bytes: usize, + pub electrum_rpc_max_response_num_bytes: usize, + pub electrum_rpc_global_response_budget_bytes: usize, pub http_addr: SocketAddr, pub http_socket_file: Option, pub monitoring_addr: SocketAddr, @@ -136,6 +139,24 @@ fn str_to_socketaddr(address: &str, what: &str) -> SocketAddr { .unwrap() } +fn value_or_default_or_exit( + m: &clap::ArgMatches, + arg_name: &str, + cli_name: &str, + default: usize, +) -> usize { + match m.value_of(arg_name) { + None => default, + Some(raw) => raw.parse::().unwrap_or_else(|_| { + clap::Error::with_description( + &format!("--{} expects a non-negative integer (got {:?})", cli_name, raw), + clap::ErrorKind::ValueValidation, + ) + .exit() + }), + } +} + impl Config { pub fn from_args() -> Config { let network_help = format!("Select network type ({})", Network::names().join(", ")); @@ -196,6 +217,13 @@ impl Config { .default_value("0") .takes_value(true), ) + .arg( + Arg::with_name("electrum_rpc_write_timeout") + .long("electrum-rpc-write-timeout") + .help("Maximum time (in seconds) to transmit a complete Electrum response, including a whole batch, errors, or a notification. Starts at the first write and is not extended by partial progress. Closes stalled connections and releases response buffers. Does not time idle subscriptions or command execution. 0 = disabled (default: 30).") + .default_value("30") + .takes_value(true), + ) .arg( Arg::with_name("electrum_rpc_max_request_num_bytes") .long("electrum-rpc-max-request-num-bytes") @@ -203,6 +231,18 @@ impl Config { .default_value("1048576") .takes_value(true), ) + .arg( + Arg::with_name("electrum_rpc_max_response_num_bytes") + .long("electrum-rpc-max-response-num-bytes") + .help("Maximum size (in bytes) of a single Electrum RPC solicited reply line, including brackets, commas, error objects, and terminating newline. Applies to singles, batches, parse errors, and invalid-request errors; subscription notifications are exempt. When a reply cannot fit, the client receives a correlated error (code 1) for the overflowing element and later batch elements are not executed. 0 = unlimited (default: 8388608, i.e. 8 MiB on Bitcoin; 33554432, i.e. 32 MiB on Liquid, sized for a 2016-header window during a dynafed parameter vote).") + .takes_value(true), + ) + .arg( + Arg::with_name("electrum_rpc_global_response_budget_bytes") + .long("electrum-rpc-global-response-budget-bytes") + .help("Aggregate cap (in bytes) on Electrum response-buffer memory retained across all connections at any instant. Bounds the memory a set of non-reading clients can hold while writer threads block in write_all. Small replies (< 16 KiB per connection) are exempt; above that floor each reply charges the budget in 64 KiB chunks (or a smaller final chunk) and releases on drop. Requests that would exceed the budget are rejected with a server error (code 2) instead of allocating. Must be >= --electrum-rpc-max-response-num-bytes so a single maximum-size reply can succeed. 0 = unlimited (default: 67108864, i.e. 64 MiB on Bitcoin; 268435456, i.e. 256 MiB on Liquid, keeping the same ratio to the per-line cap).") + .takes_value(true), + ) .arg( Arg::with_name("http_addr") .long("http-addr") @@ -457,6 +497,12 @@ impl Config { #[cfg(feature = "liquid")] Network::LiquidRegtest => 51401, }; + #[cfg(not(feature = "liquid"))] + let default_electrum_rpc_max_response_num_bytes: usize = 8_388_608; + #[cfg(feature = "liquid")] + let default_electrum_rpc_max_response_num_bytes: usize = 33_554_432; + let default_electrum_rpc_global_response_budget_bytes: usize = + default_electrum_rpc_max_response_num_bytes.saturating_mul(8); let default_http_port = match network_type { #[cfg(not(feature = "liquid"))] Network::Bitcoin => 3000, @@ -521,6 +567,10 @@ impl Config { 0 => None, // 0 = unlimited / never disconnect secs => Some(Duration::from_secs(secs)), }; + let electrum_rpc_write_timeout = match value_t_or_exit!(m, "electrum_rpc_write_timeout", u64) { + 0 => None, + secs => Some(Duration::from_secs(secs)), + }; let electrum_rpc_max_request_num_bytes: usize = match value_t_or_exit!( m, "electrum_rpc_max_request_num_bytes", @@ -529,6 +579,40 @@ impl Config { 0 => usize::MAX, // 0 = unlimited bytes => bytes, }; + let electrum_rpc_max_response_num_bytes: usize = match value_or_default_or_exit( + &m, + "electrum_rpc_max_response_num_bytes", + "electrum-rpc-max-response-num-bytes", + default_electrum_rpc_max_response_num_bytes, + ) { + 0 => usize::MAX, // 0 = unlimited + bytes => bytes, + }; + let electrum_rpc_global_response_budget_bytes: usize = match value_or_default_or_exit( + &m, + "electrum_rpc_global_response_budget_bytes", + "electrum-rpc-global-response-budget-bytes", + default_electrum_rpc_global_response_budget_bytes, + ) { + 0 => usize::MAX, // 0 = unlimited + bytes => bytes, + }; + if electrum_rpc_global_response_budget_bytes != usize::MAX + && electrum_rpc_max_response_num_bytes != usize::MAX + && electrum_rpc_global_response_budget_bytes < electrum_rpc_max_response_num_bytes + { + clap::Error::with_description( + &format!( + "--electrum-rpc-global-response-budget-bytes ({}) must be >= \ + --electrum-rpc-max-response-num-bytes ({}); a single maximum-size \ + reply cannot succeed otherwise", + electrum_rpc_global_response_budget_bytes, + electrum_rpc_max_response_num_bytes, + ), + clap::ErrorKind::ValueValidation, + ) + .exit(); + } let http_addr: SocketAddr = str_to_socketaddr( m.value_of("http_addr") .unwrap_or(&format!("127.0.0.1:{}", default_http_port)), @@ -601,7 +685,10 @@ impl Config { history_scan_limit: value_t_or_exit!(m, "history_scan_limit", usize), electrum_rpc_addr, electrum_rpc_conn_max_age, + electrum_rpc_write_timeout, electrum_rpc_max_request_num_bytes, + electrum_rpc_max_response_num_bytes, + electrum_rpc_global_response_budget_bytes, electrum_txs_limit: value_t_or_exit!(m, "electrum_txs_limit", usize), electrum_subscription_limit: value_t_or_exit!(m, "electrum_subscription_limit", usize), electrum_checkpoint_proof_concurrency_limit: value_t_or_exit!( diff --git a/src/electrum/client_io.rs b/src/electrum/client_io.rs new file mode 100644 index 000000000..5a4528dfe --- /dev/null +++ b/src/electrum/client_io.rs @@ -0,0 +1,363 @@ +//! One deadline for all writes making up a client-facing response. + +use std::io::{self, Write}; +use std::net::{Shutdown, TcpStream}; +use std::time::{Duration, Instant}; + +use crate::errors::*; +use crate::metrics::Counter; + +pub(super) fn write_to_client( + stream: &TcpStream, + timeout: Option, + timeouts: &Counter, + write: impl FnOnce(&mut ClientWriter<'_>) -> Result<()>, +) -> Result<()> { + if timeout.is_none() { + stream + .set_write_timeout(None) + .chain_err(|| "failed to disable client write timeout")?; + } + let mut writer = ClientWriter { + stream, + timeout, + deadline: None, + timed_out: false, + }; + let result = write(&mut writer); + if writer.timed_out { + timeouts.inc(); + // Preserve the classification even when serde wrapped the I/O error + // while streaming a budget rejection. + Err(ErrorKind::ClientWriteTimeout.into()) + } else { + result + } +} + +pub(super) struct ClientWriter<'a> { + stream: &'a TcpStream, + timeout: Option, + // Start at the first write, after command execution and reply assembly. + deadline: Option, + timed_out: bool, +} + +impl ClientWriter<'_> { + fn expired(&mut self) -> io::Error { + if !self.timed_out { + self.timed_out = true; + // Discard queued output on close instead of leaving a FIN behind + // bytes the client will not read. Only timed-out writes abort this way. + if let Err(error) = socket2::SockRef::from(self.stream).set_linger(Some(Duration::ZERO)) + { + warn!( + "failed to enable abortive close after client write timeout: {}", + error + ); + } + // Wake the reader; teardown closes all socket handles to send the RST. + let _ = self.stream.shutdown(Shutdown::Both); + } + io::Error::new(io::ErrorKind::TimedOut, "client response write timed out") + } +} + +impl Write for ClientWriter<'_> { + fn write(&mut self, bytes: &[u8]) -> io::Result { + if self.timed_out { + return Err(self.expired()); + } + if bytes.is_empty() { + return Ok(0); + } + let timeout = match self.timeout { + Some(timeout) => timeout, + None => return (&*self.stream).write(bytes), + }; + let now = Instant::now(); + let deadline = match self.deadline { + Some(deadline) => deadline, + None => { + let deadline = now.checked_add(timeout).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "client write timeout is too large", + ) + })?; + self.deadline = Some(deadline); + deadline + } + }; + if now >= deadline { + return Err(self.expired()); + } + + // A fresh full timeout on each partial write would let a trickling + // reader extend the response indefinitely. Only use the remaining time. + self.stream.set_write_timeout(Some(deadline - now))?; + let result = (&*self.stream).write(bytes); + if Instant::now() >= deadline + || matches!(&result, Err(error) if matches!(error.kind(), io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut)) + { + return Err(self.expired()); + } + result + } + + fn flush(&mut self) -> io::Result<()> { + if self.timed_out + || self + .deadline + .map_or(false, |deadline| Instant::now() >= deadline) + { + return Err(self.expired()); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Read; + use std::net::TcpListener; + use std::sync::{mpsc, Arc}; + use std::thread; + + // A watchdog bounds the test even if the write deadline regresses. + struct SocketPair { + server: Arc, + client: TcpStream, + stop: mpsc::Sender<()>, + watchdog: Option>, + } + + impl SocketPair { + fn new() -> Self { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let client = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + let (server, _) = listener.accept().unwrap(); + let server = Arc::new(server); + socket2::SockRef::from(&*server) + .set_send_buffer_size(4096) + .unwrap(); + client + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let (stop, receiver) = mpsc::channel(); + let socket = Arc::clone(&server); + let watchdog = thread::spawn(move || { + if receiver.recv_timeout(Duration::from_secs(5)).is_err() { + let _ = socket.shutdown(Shutdown::Both); + } + }); + Self { + server, + client, + stop, + watchdog: Some(watchdog), + } + } + } + + impl Drop for SocketPair { + fn drop(&mut self) { + let _ = self.server.shutdown(Shutdown::Both); + let _ = self.client.shutdown(Shutdown::Both); + let _ = self.stop.send(()); + self.watchdog.take().unwrap().join().unwrap(); + } + } + + fn counter() -> Counter { + Counter::new("client_write_timeouts", "Test timeouts").unwrap() + } + + #[test] + fn stalled_writer_times_out_without_the_client_disconnecting() { + let sockets = SocketPair::new(); + socket2::SockRef::from(&sockets.client) + .set_recv_buffer_size(4096) + .unwrap(); + let timeouts = counter(); + let start = Instant::now(); + let result = write_to_client( + &sockets.server, + Some(Duration::from_millis(150)), + &timeouts, + |writer| { + writer + .write_all(&vec![b'x'; 2 * 1024 * 1024]) + .chain_err(|| "write failed") + }, + ); + assert!(matches!( + result.unwrap_err().kind(), + ErrorKind::ClientWriteTimeout + )); + assert!(start.elapsed() < Duration::from_secs(3)); + assert_eq!(timeouts.get(), 1); + // sockets.client is still open and has never read anything. + } + + #[test] + fn reading_client_receives_payload_and_newline() { + let mut sockets = SocketPair::new(); + let timeouts = counter(); + write_to_client( + &sockets.server, + Some(Duration::from_secs(1)), + &timeouts, + |writer| { + writer.write_all(b"hello").chain_err(|| "payload failed")?; + writer.write_all(b"\n").chain_err(|| "newline failed") + }, + ) + .unwrap(); + let mut bytes = [0; 6]; + sockets.client.read_exact(&mut bytes).unwrap(); + assert_eq!(&bytes, b"hello\n"); + assert_eq!(timeouts.get(), 0); + } + + #[test] + fn delayed_reader_can_finish_and_disabled_timeout_clears_the_socket_limit() { + for timeout in [Some(Duration::from_secs(4)), None] { + let sockets = SocketPair::new(); + sockets + .server + .set_write_timeout(Some(Duration::from_millis(10))) + .unwrap(); + let mut client = sockets.client.try_clone().unwrap(); + let reader = thread::spawn(move || { + thread::sleep(Duration::from_millis(150)); + let mut bytes = Vec::new(); + client.read_to_end(&mut bytes).unwrap(); + bytes + }); + let timeouts = counter(); + let payload = vec![b'x'; 512 * 1024]; + let result = write_to_client(&sockets.server, timeout, &timeouts, |writer| { + writer.write_all(&payload).chain_err(|| "write failed") + }); + let _ = sockets.server.shutdown(Shutdown::Write); + let received = reader.join().unwrap(); + result.unwrap(); + assert_eq!(received, payload); + assert_eq!(timeouts.get(), 0); + } + } + + #[test] + fn chunks_and_newline_share_one_deadline() { + let sockets = SocketPair::new(); + let timeouts = counter(); + let result = write_to_client( + &sockets.server, + Some(Duration::from_secs(1)), + &timeouts, + |writer| { + assert!( + writer.deadline.is_none(), + "assembly must not start the write clock" + ); + writer + .write_all(b"payload") + .chain_err(|| "payload failed")?; + assert!(writer.deadline.is_some()); + // Advance the deadline explicitly instead of racing a short sleep. + writer.deadline = Some(Instant::now() - Duration::from_secs(1)); + let newline = writer.write_all(b"\n").chain_err(|| "newline failed"); + assert!(writer.write_all(b"retry").is_err()); + newline + }, + ); + assert!(matches!( + result.unwrap_err().kind(), + ErrorKind::ClientWriteTimeout + )); + assert_eq!(timeouts.get(), 1); + } + + #[test] + fn partial_progress_does_not_extend_the_deadline() { + let sockets = SocketPair::new(); + socket2::SockRef::from(&sockets.client) + .set_recv_buffer_size(4096) + .unwrap(); + let mut client = sockets.client.try_clone().unwrap(); + let (stop, receiver) = mpsc::channel(); + let reader = thread::spawn(move || { + let mut total = 0; + let mut bytes = [0; 1024]; + loop { + match client.read(&mut bytes) { + Ok(0) | Err(_) => break, + Ok(n) => total += n, + } + if receiver.recv_timeout(Duration::from_millis(10)).is_ok() { + break; + } + } + total + }); + let timeouts = counter(); + let start = Instant::now(); + let result = write_to_client( + &sockets.server, + Some(Duration::from_millis(250)), + &timeouts, + |writer| { + writer + .write_all(&vec![b'x'; 2 * 1024 * 1024]) + .chain_err(|| "write failed") + }, + ); + let _ = stop.send(()); + assert!(reader.join().unwrap() > 0); + assert!(matches!( + result.unwrap_err().kind(), + ErrorKind::ClientWriteTimeout + )); + assert!(start.elapsed() < Duration::from_secs(3)); + assert_eq!(timeouts.get(), 1); + } + + #[test] + fn streaming_budget_errors_and_notifications_use_the_same_write_deadline() { + use crate::electrum::response::{ResponseBudget, ResponseWriter}; + + for notification in [false, true] { + let sockets = SocketPair::new(); + socket2::SockRef::from(&sockets.client) + .set_recv_buffer_size(4096) + .unwrap(); + let timeouts = counter(); + let mut responses = ResponseWriter::new(2 * 1024 * 1024, ResponseBudget::new(1)); + let result = write_to_client( + &sockets.server, + Some(Duration::from_millis(150)), + &timeouts, + |writer| { + if notification { + responses + .send_notification(writer, &json!({"params":["x".repeat(1024 * 1024)]})) + } else { + responses.send( + writer, + Ok(json!({"id":"x".repeat(1024 * 1024), "method":"server.ping"})), + |_| panic!("budget rejection must not execute the command"), + |_| {}, + ) + } + }, + ); + assert!(matches!( + result.unwrap_err().kind(), + ErrorKind::ClientWriteTimeout + )); + assert_eq!(timeouts.get(), 1); + } + } +} diff --git a/src/electrum/mod.rs b/src/electrum/mod.rs index d28426d7d..499b9c241 100644 --- a/src/electrum/mod.rs +++ b/src/electrum/mod.rs @@ -1,3 +1,5 @@ +mod client_io; +mod response; mod server; pub use server::RPC; diff --git a/src/electrum/response.rs b/src/electrum/response.rs new file mode 100644 index 000000000..a1847d213 --- /dev/null +++ b/src/electrum/response.rs @@ -0,0 +1,934 @@ +use std::io::{self, BufWriter, Write}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use serde::Serialize; +use serde_json::Value; + +use crate::errors::*; + +const MAX_ARRAY_BATCH: usize = 20; +const BUFFER_FLOOR: usize = 16 * 1024; +const BUDGET_CHUNK: usize = 64 * 1024; +const PER_LINE_MSG: &str = "response line exceeds max_response_bytes"; +const GLOBAL_MSG: &str = "response budget exhausted, retry later"; +const SKIPPED_MSG: &str = "batch response budget exhausted; not executed"; + +const _: () = { + assert!(SKIPPED_MSG.len() >= PER_LINE_MSG.len()); + assert!(SKIPPED_MSG.len() >= GLOBAL_MSG.len()); +}; + +// JSON-RPC errors plus Electrum application error codes. +#[repr(i16)] +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) enum JsonRpcV2Error { + ParseError = -32700, + InvalidRequest = -32600, + MethodNotFound = -32601, + InvalidParams = -32602, + InternalError = -32603, + BadRequest = 1, + DaemonError = 2, +} + +impl JsonRpcV2Error { + pub(super) fn into_i16(self) -> i16 { + self as i16 + } +} + +#[derive(Serialize)] +struct ErrorEnvelope<'a> { + error: ErrorBody<'a>, + id: &'a Value, + jsonrpc: &'static str, +} + +#[derive(Serialize)] +struct ErrorBody<'a> { + code: i16, + message: &'a str, +} + +impl<'a> ErrorEnvelope<'a> { + fn new(message: &'a str, id: Option<&'a Value>, code: JsonRpcV2Error) -> Self { + Self { + error: ErrorBody { + code: code.into_i16(), + message, + }, + id: id.unwrap_or(&Value::Null), + jsonrpc: "2.0", + } + } + + fn encoded_len(&self) -> usize { + let mut counter = ByteCounter(0); + serde_json::to_writer(&mut counter, self).expect("counting JSON values cannot fail"); + counter.0 + } +} + +pub(super) fn json_rpc_error( + message: impl std::fmt::Display, + id: Option<&Value>, + code: JsonRpcV2Error, +) -> Value { + serde_json::to_value(ErrorEnvelope::new(&message.to_string(), id, code)) + .expect("an error envelope contains only JSON values") +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum ResponseLimit { + PerLine, + Global, +} + +impl ResponseLimit { + fn message(self) -> &'static str { + match self { + Self::PerLine => PER_LINE_MSG, + Self::Global => GLOBAL_MSG, + } + } + + fn code(self) -> JsonRpcV2Error { + match self { + Self::PerLine => JsonRpcV2Error::BadRequest, + Self::Global => JsonRpcV2Error::DaemonError, + } + } + + fn error(self) -> Error { + match self { + Self::PerLine => ErrorKind::PerLineResponseOverflow.into(), + Self::Global => ErrorKind::ResponseBudgetExhausted.into(), + } + } +} + +pub(super) struct ResponseBudget { + limit: usize, + reserved: AtomicUsize, +} + +impl ResponseBudget { + pub(super) fn new(limit: usize) -> Arc { + Arc::new(Self { + limit, + reserved: AtomicUsize::new(0), + }) + } +} + +struct Reservation { + budget: Arc, + granted: usize, +} + +impl Reservation { + fn ensure(&mut self, total: usize) -> std::result::Result<(), ResponseLimit> { + if self.budget.limit == usize::MAX || total <= self.granted { + return Ok(()); + } + let required = total - self.granted; + let preferred = required + .checked_next_multiple_of(BUDGET_CHUNK) + .unwrap_or(required); + let mut reserved = self.budget.reserved.load(Ordering::Relaxed); + loop { + let available = self.budget.limit - reserved; + if required > available { + return Err(ResponseLimit::Global); + } + let grant = preferred.min(available); + match self.budget.reserved.compare_exchange_weak( + reserved, + reserved + grant, + Ordering::AcqRel, + Ordering::Relaxed, + ) { + Ok(_) => { + self.granted += grant; + return Ok(()); + } + Err(latest) => reserved = latest, + } + } + } +} + +impl Drop for Reservation { + fn drop(&mut self) { + self.budget + .reserved + .fetch_sub(self.granted, Ordering::AcqRel); + } +} + +// Fields drop in declaration order: free the buffer before releasing its grant. +struct ResponseBuffer { + bytes: Vec, + reservation: Reservation, + limit: usize, +} + +impl ResponseBuffer { + fn new(bytes: Vec, budget: Arc, limit: usize) -> Self { + Self { + bytes, + reservation: Reservation { budget, granted: 0 }, + limit, + } + } + + fn reserve(&mut self, line_bytes: usize) -> std::result::Result<(), ResponseLimit> { + self.reservation + .ensure(line_bytes.saturating_sub(BUFFER_FLOOR)) + } + + fn encode( + &mut self, + value: &impl Serialize, + tail: usize, + ) -> std::result::Result<(), EncodeError> { + let limit = self + .limit + .checked_sub(tail) + .ok_or(EncodeError::Limited(ResponseLimit::PerLine))?; + let start = self.bytes.len(); + let mut writer = LimitedWriter { + buffer: self, + limit, + tail, + failure: None, + }; + match serde_json::to_writer(&mut writer, value) { + Ok(()) => Ok(()), + Err(error) => { + writer.buffer.bytes.truncate(start); + Err(match writer.failure { + Some(limit) => EncodeError::Limited(limit), + None => EncodeError::Json(error), + }) + } + } + } + + fn framing(&mut self, bytes: &[u8]) -> Result<()> { + let limit = self.limit; + let mut writer = LimitedWriter { + buffer: self, + limit, + tail: 0, + failure: None, + }; + writer + .write_all(bytes) + .map_err(|_| writer.failure.expect("bounded write failed").error()) + } + + fn recycle(mut self) -> Vec { + cleanup_buffer(&mut self.bytes); + std::mem::take(&mut self.bytes) + } +} + +enum EncodeError { + Limited(ResponseLimit), + Json(serde_json::Error), +} + +impl From for Error { + fn from(error: EncodeError) -> Self { + match error { + EncodeError::Limited(limit) => limit.error(), + EncodeError::Json(error) => Error::with_chain(error, "failed to serialize response"), + } + } +} + +struct LimitedWriter<'a> { + buffer: &'a mut ResponseBuffer, + limit: usize, + tail: usize, + failure: Option, +} + +impl Write for LimitedWriter<'_> { + fn write(&mut self, bytes: &[u8]) -> io::Result { + if bytes.is_empty() { + return Ok(0); + } + let result = self + .buffer + .bytes + .len() + .checked_add(bytes.len()) + .filter(|len| *len <= self.limit) + .ok_or(ResponseLimit::PerLine) + .and_then(|len| { + self.buffer.reserve(len.saturating_add(self.tail))?; + Ok(len) + }); + let len = match result { + Ok(len) => len, + Err(limit) => { + self.failure = Some(limit); + return Err(io::Error::new(io::ErrorKind::Other, limit.message())); + } + }; + if len > self.buffer.bytes.capacity() { + // Avoid Vec's implicit doubling: every byte above the exempt floor + // must be charged, including capacity retained after a rollback. + let capacity = if len <= BUFFER_FLOOR { + BUFFER_FLOOR + } else if self.buffer.reservation.budget.limit == usize::MAX { + len.checked_next_multiple_of(BUDGET_CHUNK).unwrap_or(len) + } else { + self.buffer.reservation.granted.saturating_add(BUFFER_FLOOR) + } + .min(self.limit); + self.buffer + .bytes + .reserve_exact(capacity - self.buffer.bytes.len()); + } + self.buffer.bytes.extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +struct ByteCounter(usize); + +impl Write for ByteCounter { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.0 = self.0.saturating_add(bytes.len()); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +enum Delivery { + Buffered, + BudgetErrors { ids: Vec, batch: bool }, +} + +impl Delivery { + fn budget_errors(requests: Vec, batch: bool) -> Self { + let ids = requests + .into_iter() + .map(|mut request| { + request + .as_object_mut() + .and_then(|object| object.remove("id")) + .unwrap_or(Value::Null) + }) + .collect(); + Self::BudgetErrors { ids, batch } + } + + fn send(self, stream: &mut impl Write, buffer: &ResponseBuffer) -> Result<()> { + match self { + Self::Buffered => stream + .write_all(&buffer.bytes) + .chain_err(|| format!("failed to write {} response bytes", buffer.bytes.len())), + Self::BudgetErrors { ids, batch } => { + let mut writer = BufWriter::with_capacity(BUFFER_FLOOR, stream); + if batch { + writer + .write_all(b"[") + .chain_err(|| "failed to send batch framing")?; + } + for (i, id) in ids.iter().enumerate() { + if i > 0 { + writer + .write_all(b",") + .chain_err(|| "failed to send batch framing")?; + } + let error = + ErrorEnvelope::new(GLOBAL_MSG, Some(id), JsonRpcV2Error::DaemonError); + serde_json::to_writer(&mut writer, &error) + .chain_err(|| "failed to send budget error")?; + } + writer + .write_all(if batch { b"]\n" } else { b"\n" }) + .chain_err(|| "failed to send response framing")?; + writer.flush().chain_err(|| "failed to send budget errors") + } + } + } +} + +pub(super) struct ResponseWriter { + buffer: Vec, + budget: Arc, + limit: usize, +} + +impl ResponseWriter { + pub(super) fn new(limit: usize, budget: Arc) -> Self { + Self { + buffer: Vec::new(), + budget, + limit, + } + } + + pub(super) fn send( + &mut self, + stream: &mut impl Write, + request: serde_json::Result, + mut execute: impl FnMut(&Value) -> Value, + mut rejected: impl FnMut(ResponseLimit), + ) -> Result<()> { + let mut buffer = ResponseBuffer::new( + std::mem::take(&mut self.buffer), + Arc::clone(&self.budget), + self.limit, + ); + let assembled = match request { + Ok(Value::Array(commands)) => { + assemble_batch(commands, &mut buffer, &mut execute, &mut rejected) + } + Ok(command) => assemble_single(command, &mut buffer, &mut execute, &mut rejected), + Err(_) => { + let error = ErrorEnvelope::new("parse error", None, JsonRpcV2Error::ParseError); + buffer + .encode(&error, 1) + .map_err(|error| { + if let EncodeError::Limited(limit) = &error { + rejected(*limit); + } + Error::from(error) + }) + .and_then(|()| buffer.framing(b"\n")) + .map(|()| Delivery::Buffered) + } + }; + let result = assembled.and_then(|delivery| delivery.send(stream, &buffer)); + self.buffer = buffer.recycle(); + result + } + + // Notifications are exempt from byte limits; the caller still bounds writes. + pub(super) fn send_notification( + &mut self, + stream: &mut impl Write, + value: &Value, + ) -> Result<()> { + let result = serde_json::to_writer(&mut self.buffer, value) + .chain_err(|| "failed to serialize notification") + .and_then(|()| { + self.buffer.push(b'\n'); + stream + .write_all(&self.buffer) + .chain_err(|| "failed to send notification") + }); + cleanup_buffer(&mut self.buffer); + result + } +} + +fn try_reserve_single_error_response( + buffer: &mut ResponseBuffer, + id: Option<&Value>, + rejected: &mut impl FnMut(ResponseLimit), +) -> Result { + let error_size = ErrorEnvelope::new(PER_LINE_MSG, id, JsonRpcV2Error::BadRequest) + .encoded_len() + .checked_add(1) + .ok_or_else(|| ResponseLimit::PerLine.error())?; + if let Err(limit) = buffer.reserve(error_size) { + rejected(limit); + if error_size > buffer.limit { + return Err(ResponseLimit::PerLine.error()); + } + buffer.bytes = Vec::new(); + return Ok(false); + } + Ok(true) +} + +fn assemble_single( + command: Value, + buffer: &mut ResponseBuffer, + execute: &mut impl FnMut(&Value) -> Value, + rejected: &mut impl FnMut(ResponseLimit), +) -> Result { + if buffer.limit == 0 { + return Err(ResponseLimit::PerLine.error()); + } + let id = command.get("id"); + if !try_reserve_single_error_response(buffer, id, rejected)? { + return Ok(Delivery::budget_errors(vec![command], false)); + } + match buffer.encode(&execute(&command), 1) { + Ok(()) => {} + Err(EncodeError::Limited(limit)) => { + rejected(limit); + buffer.encode(&ErrorEnvelope::new(limit.message(), id, limit.code()), 1)?; + } + Err(error) => return Err(error.into()), + } + buffer.framing(b"\n")?; + Ok(Delivery::Buffered) +} + +fn try_reserve_batch_error_responses( + commands: &[Value], + buffer: &mut ResponseBuffer, + rejected: &mut impl FnMut(ResponseLimit), +) -> Result> { + let n = commands.len(); + let mut error_sizes = [0; MAX_ARRAY_BATCH]; + let mut later_errors = 0usize; + for (command, size) in commands.iter().zip(error_sizes.iter_mut()) { + *size = ErrorEnvelope::new(SKIPPED_MSG, command.get("id"), JsonRpcV2Error::BadRequest) + .encoded_len(); + later_errors = later_errors + .checked_add(*size) + .ok_or("batch reserve overflow")?; + } + // '[' + (n - 1) commas + ']\n'. + let all_errors_size = later_errors + .checked_add(n + 2) + .ok_or("batch framing overflow")?; + if all_errors_size > buffer.limit { + bail!( + "cannot fit correlated error envelopes for batch of {} \ + under electrum_rpc_max_response_num_bytes={}", + n, + buffer.limit + ); + } + if let Err(limit) = buffer.reserve(all_errors_size) { + rejected(limit); + buffer.bytes = Vec::new(); + return Ok(None); + } + + Ok(Some(error_sizes)) +} + +fn assemble_batch( + commands: Vec, + buffer: &mut ResponseBuffer, + execute: &mut impl FnMut(&Value) -> Value, + rejected: &mut impl FnMut(ResponseLimit), +) -> Result { + let n = commands.len(); + if n > MAX_ARRAY_BATCH { + bail!( + "Too many elements in batch requests {} max:{}", + n, + MAX_ARRAY_BATCH + ); + } + if n == 0 { + buffer.framing(b"[]\n")?; + return Ok(Delivery::Buffered); + } + + let Some(error_sizes) = try_reserve_batch_error_responses(&commands, buffer, rejected)? else { + return Ok(Delivery::budget_errors(commands, true)); + }; + let mut later_errors = error_sizes.iter().sum::(); + + buffer.framing(b"[")?; + let mut overflow: Option = None; + for (i, command) in commands.into_iter().enumerate() { + if i > 0 { + buffer.framing(b",")?; + } + later_errors -= error_sizes[i]; + let tail = later_errors + (n - 1 - i) + b"]\n".len(); + let (message, code) = match overflow { + Some(limit) => (SKIPPED_MSG, limit.code()), + None => match buffer.encode(&execute(&command), tail) { + Ok(()) => continue, + Err(EncodeError::Limited(limit)) => { + rejected(limit); + overflow = Some(limit); + (limit.message(), limit.code()) + } + Err(error) => return Err(error.into()), + }, + }; + buffer.encode(&ErrorEnvelope::new(message, command.get("id"), code), tail)?; + } + buffer.framing(b"]\n")?; + Ok(Delivery::Buffered) +} + +fn cleanup_buffer(buffer: &mut Vec) { + buffer.clear(); + if buffer.capacity() > BUFFER_FLOOR { + *buffer = Vec::new(); + } +} + +pub(super) fn warn_limits(max_request_bytes: usize, max_response_bytes: usize, txs_limit: usize) { + if max_response_bytes == usize::MAX { + return; + } + let worst_reserve = max_request_bytes + .saturating_add(MAX_ARRAY_BATCH * 128) + .saturating_add(MAX_ARRAY_BATCH + 2); + if worst_reserve >= max_response_bytes { + warn!( + "electrum_rpc_max_response_num_bytes={} is not large enough to guarantee \ + correlated error replies for a full batch of requests up to \ + electrum_rpc_max_request_num_bytes={}; such batches will be rejected by \ + closing the connection", + max_response_bytes, max_request_bytes + ); + } + const HISTORY_ENTRY_ESTIMATE_BYTES: usize = 128; + if txs_limit.saturating_mul(HISTORY_ENTRY_ESTIMATE_BYTES) >= max_response_bytes { + warn!( + "electrum_txs_limit={} can generate histories exceeding \ + electrum_rpc_max_response_num_bytes={} (rough estimate: {} bytes per \ + entry, IDs/UTXOs/transactions/headers excluded); legitimate history \ + replies can be rejected under this configuration", + txs_limit, max_response_bytes, HISTORY_ENTRY_ESTIMATE_BYTES + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Barrier; + + fn send_reply( + limit: usize, + budget: usize, + reply: &Value, + ) -> (Value, usize, Vec) { + let mut writer = ResponseWriter::new(limit, ResponseBudget::new(budget)); + let mut bytes = Vec::new(); + let mut rejected = Vec::new(); + writer + .send( + &mut bytes, + Ok(json!({"id": reply["id"]})), + |_| reply.clone(), + |limit| rejected.push(limit), + ) + .unwrap(); + assert!(bytes.ends_with(b"\n")); + assert_eq!(writer.budget.reserved.load(Ordering::Relaxed), 0); + assert!(writer.buffer.capacity() <= BUFFER_FLOOR); + ( + serde_json::from_slice(&bytes).unwrap(), + bytes.len(), + rejected, + ) + } + + #[test] + fn single_limit_includes_escaped_id_and_newline() { + let reply = json!({"id": "a\n\"\\é", "jsonrpc": "2.0", "result": "x".repeat(256)}); + let exact = serde_json::to_vec(&reply).unwrap().len() + 1; + let (actual, size, rejected) = send_reply(exact, usize::MAX, &reply); + assert_eq!(actual, reply); + assert_eq!(size, exact); + assert!(rejected.is_empty()); + + let (actual, size, rejected) = send_reply(exact - 1, usize::MAX, &reply); + assert_eq!(actual["id"], reply["id"]); + assert_eq!(actual["error"]["code"], 1); + assert!(size < exact); + assert_eq!(rejected, [ResponseLimit::PerLine]); + } + + #[test] + fn partial_final_grants_allow_replies_below_both_limits() { + for limit in [32_768, 100_000] { + let reply = json!({"id": 1, "result": "x".repeat(limit - 100)}); + let (actual, size, rejected) = send_reply(limit, limit, &reply); + assert_eq!(actual, reply); + assert!(size <= limit); + assert!(rejected.is_empty()); + } + } + + #[test] + fn framing_and_rollback_keep_buffer_capacity_charged() { + for suffix in [b"\n".as_slice(), b"]\n".as_slice(), b",".as_slice()] { + let budget = ResponseBudget::new(100_000); + let mut buffer = ResponseBuffer::new(Vec::new(), Arc::clone(&budget), 100_000); + buffer + .encode(&json!("x".repeat(90_000)), suffix.len()) + .map_err(Error::from) + .unwrap(); + assert!(buffer + .encode(&json!("y".repeat(20_000)), suffix.len()) + .is_err()); + buffer.framing(suffix).unwrap(); + assert!(buffer.bytes.capacity() <= buffer.reservation.granted + BUFFER_FLOOR); + assert!(buffer.bytes.ends_with(suffix)); + assert_eq!(buffer.bytes.len(), 90_002 + suffix.len()); + drop(buffer); + assert_eq!(budget.reserved.load(Ordering::Relaxed), 0); + } + } + + #[test] + fn batch_error_uses_reserved_tail_when_no_more_budget_is_available() { + let budget = ResponseBudget::new(BUDGET_CHUNK); + let mut buffer = ResponseBuffer::new(Vec::new(), Arc::clone(&budget), 100_000); + let delivery = assemble_batch( + vec![json!({"id":1}), json!({"id":"b".repeat(1000)})], + &mut buffer, + &mut |command: &Value| { + if command["id"] == 1 { + json!({"id":1, "result":"a".repeat(80_000)}) + } else { + json!({"id":command["id"], "result":"c".repeat(10_000)}) + } + }, + &mut |limit| assert_eq!(limit, ResponseLimit::Global), + ) + .unwrap(); + assert_eq!(budget.reserved.load(Ordering::Relaxed), BUDGET_CHUNK); + assert!(buffer.bytes.capacity() <= BUDGET_CHUNK + BUFFER_FLOOR); + let mut bytes = Vec::new(); + delivery.send(&mut bytes, &buffer).unwrap(); + let response: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(response[0]["result"].as_str().unwrap().len(), 80_000); + assert_eq!(response[1]["id"], "b".repeat(1000)); + assert_eq!(response[1]["error"]["code"], 2); + drop(buffer); + assert_eq!(budget.reserved.load(Ordering::Relaxed), 0); + } + + #[test] + fn batch_reserves_serialized_errors_before_executing_later_commands() { + let first = json!({"id": 1, "jsonrpc": "2.0", "result": "x".repeat(1000)}); + let second_id = "a\n\"\\é".repeat(10); + let skipped = |id: Value| { + json!({ + "jsonrpc": "2.0", "id": id, + "error": {"code": 1, "message": "batch response budget exhausted; not executed"} + }) + }; + let reserved_line = json!([first, skipped(json!(second_id)), skipped(json!(3))]); + let exact = serde_json::to_vec(&reserved_line).unwrap().len() + 1; + for cap in [exact, exact - 1] { + let mut writer = ResponseWriter::new(cap, ResponseBudget::new(usize::MAX)); + let mut calls = Vec::new(); + let mut bytes = Vec::new(); + writer + .send( + &mut bytes, + Ok(json!([{"id":1}, {"id":second_id}, {"id":3}])), + |cmd| { + calls.push(cmd["id"].clone()); + if cmd["id"] == 1 { + first.clone() + } else { + json!({"id": cmd["id"], "result": "y".repeat(cap)}) + } + }, + |limit| assert_eq!(limit, ResponseLimit::PerLine), + ) + .unwrap(); + let actual: Value = serde_json::from_slice(&bytes).unwrap(); + assert!(bytes.len() <= cap); + assert_eq!(actual[1]["id"], second_id); + assert_eq!(actual[2], skipped(json!(3))); + if cap == exact { + assert_eq!(actual[0], first); + assert_eq!(calls, [json!(1), json!(second_id)]); + assert_eq!(actual[1]["error"]["code"], 1); + } else { + assert_eq!(actual[0]["error"]["code"], 1); + assert_eq!(calls, [json!(1)]); + } + } + } + + #[test] + fn fallback_moves_only_ids_and_streams_without_a_grant() { + for batch in [false, true] { + let id = "x".repeat(32 * 1024); + let command = json!({"id": Value::String(id), "params": ["p".repeat(100_000)]}); + let allocation = command["id"].as_str().unwrap().as_ptr(); + let budget = ResponseBudget::new(1); + let mut buffer = ResponseBuffer::new(Vec::new(), Arc::clone(&budget), 100_000); + let mut execute = |_: &Value| panic!("rejected command executed"); + let mut rejected = |limit| assert_eq!(limit, ResponseLimit::Global); + let delivery = if batch { + assemble_batch(vec![command], &mut buffer, &mut execute, &mut rejected) + } else { + assemble_single(command, &mut buffer, &mut execute, &mut rejected) + } + .unwrap(); + match &delivery { + Delivery::BudgetErrors { + ids, + batch: is_batch, + } => { + assert_eq!(*is_batch, batch); + assert_eq!(ids.len(), 1); + assert_eq!(ids[0].as_str().unwrap().as_ptr(), allocation); + } + Delivery::Buffered => panic!("expected streaming fallback"), + } + assert_eq!(buffer.bytes.capacity(), 0); + assert_eq!(budget.reserved.load(Ordering::Relaxed), 0); + let mut bytes = Vec::new(); + delivery.send(&mut bytes, &buffer).unwrap(); + let response: Value = serde_json::from_slice(&bytes).unwrap(); + let response = if batch { &response[0] } else { &response }; + assert_eq!(response["id"].as_str().unwrap().len(), 32 * 1024); + assert_eq!(response["error"]["code"], 2); + } + } + + #[test] + fn budget_is_shared_until_reservations_drop_and_servers_are_independent() { + let budget = ResponseBudget::new(2 * BUDGET_CHUNK); + let reserve = || Reservation { + budget: Arc::clone(&budget), + granted: 0, + }; + let mut first = reserve(); + let mut second = reserve(); + first.ensure(BUDGET_CHUNK).unwrap(); + second.ensure(BUDGET_CHUNK).unwrap(); + let mut third = reserve(); + assert_eq!(third.ensure(1), Err(ResponseLimit::Global)); + let mut independent = Reservation { + budget: ResponseBudget::new(BUDGET_CHUNK), + granted: 0, + }; + independent.ensure(BUDGET_CHUNK).unwrap(); + drop(first); + third.ensure(BUDGET_CHUNK).unwrap(); + drop(second); + drop(third); + assert_eq!(budget.reserved.load(Ordering::Relaxed), 0); + } + + #[test] + fn concurrent_reservations_never_exceed_the_budget() { + const THREADS: usize = 8; + const GRANTS: usize = 4; + for _ in 0..16 { + let budget = ResponseBudget::new(GRANTS * BUDGET_CHUNK); + let barrier = Arc::new(Barrier::new(THREADS)); + let handles: Vec<_> = (0..THREADS) + .map(|_| { + let budget = Arc::clone(&budget); + let barrier = Arc::clone(&barrier); + std::thread::spawn(move || { + let mut reservation = Reservation { budget, granted: 0 }; + barrier.wait(); + let admitted = reservation.ensure(BUDGET_CHUNK).is_ok(); + // Keep grants alive until every thread has attempted admission. + barrier.wait(); + admitted + }) + }) + .collect(); + let admitted = handles + .into_iter() + .map(|handle| usize::from(handle.join().unwrap())) + .sum::(); + assert_eq!(admitted, GRANTS); + assert_eq!(budget.reserved.load(Ordering::Relaxed), 0); + } + } + + #[test] + fn transmission_keeps_the_grant_and_releases_it_on_error_or_panic() { + struct FailingSink { + budget: Arc, + panic: bool, + } + impl Write for FailingSink { + fn write(&mut self, _: &[u8]) -> io::Result { + assert!(self.budget.reserved.load(Ordering::Relaxed) > 0); + if self.panic { + panic!("simulated writer panic"); + } + Err(io::Error::new(io::ErrorKind::BrokenPipe, "closed")) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + for panic in [false, true] { + let budget = ResponseBudget::new(100_000); + let mut writer = ResponseWriter::new(100_000, Arc::clone(&budget)); + let mut sink = FailingSink { + budget: Arc::clone(&budget), + panic, + }; + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + writer.send( + &mut sink, + Ok(json!({"id":1})), + |_| json!({"id":1, "result":"x".repeat(90_000)}), + |_| panic!("unexpected rejection"), + ) + })); + if panic { + assert!(result.is_err()); + } else { + assert!(result.unwrap().is_err()); + } + assert_eq!(budget.reserved.load(Ordering::Relaxed), 0); + assert!(writer.buffer.capacity() <= BUFFER_FLOOR); + } + } + + #[test] + fn parse_errors_empty_batches_and_notifications_keep_their_wire_behavior() { + let mut writer = ResponseWriter::new(128, ResponseBudget::new(1)); + let mut bytes = Vec::new(); + writer + .send( + &mut bytes, + serde_json::from_str("{"), + |_| panic!("parse error executed"), + |_| panic!("unexpected rejection"), + ) + .unwrap(); + let error: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!( + error, + json!({"jsonrpc":"2.0", "id":null, "error":{"code":-32700,"message":"parse error"}}) + ); + bytes.clear(); + writer + .send( + &mut bytes, + Ok(json!([])), + |_| panic!("empty batch executed"), + |_| panic!("unexpected rejection"), + ) + .unwrap(); + assert_eq!(bytes, b"[]\n"); + bytes.clear(); + let notification = json!({"method":"test", "params":["x".repeat(100_000)]}); + writer + .send_notification(&mut bytes, ¬ification) + .unwrap(); + assert_eq!( + serde_json::from_slice::(&bytes).unwrap(), + notification + ); + assert_eq!(writer.budget.reserved.load(Ordering::Relaxed), 0); + assert!(writer.buffer.capacity() <= BUFFER_FLOOR); + } +} diff --git a/src/electrum/server.rs b/src/electrum/server.rs index aa1dc3377..39745567b 100644 --- a/src/electrum/server.rs +++ b/src/electrum/server.rs @@ -1,6 +1,6 @@ use std::cmp::Ordering; use std::collections::{BinaryHeap, HashMap}; -use std::io::{BufRead, BufReader, Write}; +use std::io::{BufRead, BufReader}; use std::net::{Shutdown, SocketAddr, TcpListener, TcpStream}; use std::sync::mpsc::{self, Receiver, RecvTimeoutError, Sender, SyncSender, TrySendError}; use std::sync::{Arc, Mutex, RwLock, Weak}; @@ -21,9 +21,13 @@ use bitcoin::consensus::encode::serialize_hex; use elements::encode::serialize_hex; use crate::chain::Txid; use crate::config::{Config, RpcLogging}; +use crate::electrum::client_io::write_to_client; +use crate::electrum::response::{ + json_rpc_error, warn_limits, JsonRpcV2Error, ResponseBudget, ResponseLimit, ResponseWriter, +}; use crate::electrum::{get_electrum_height, ProtocolVersion}; use crate::errors::*; -use crate::metrics::{Gauge, HistogramOpts, HistogramVec, MetricOpts, Metrics}; +use crate::metrics::{Counter, Gauge, HistogramOpts, HistogramVec, MetricOpts, Metrics}; use crate::new_index::{Query, Utxo}; use crate::util::electrum_merkle::{get_header_merkle_proof, get_id_from_pos, get_tx_merkle_proof}; use crate::util::{create_socket, spawn_thread, BlockId, BoolThen, Channel, FullHash, HeaderEntry}; @@ -31,8 +35,6 @@ use crate::util::{create_socket, spawn_thread, BlockId, BoolThen, Channel, FullH const ELECTRS_VERSION: &str = env!("CARGO_PKG_VERSION"); const PROTOCOL_VERSION: ProtocolVersion = ProtocolVersion::new(1, 4); const MAX_HEADERS: usize = 2016; -const MAX_ARRAY_BATCH: usize = 20; - #[cfg(feature = "electrum-discovery")] use crate::electrum::{DiscoveryManager, ServerFeatures}; @@ -82,27 +84,6 @@ fn bool_from_value_or(val: Option<&Value>, name: &str, default: bool) -> Result< bool_from_value(val, name) } -// JSON-RPC 2.0 error codes (https://www.jsonrpc.org/specification#error_object), -// plus the application-level codes used by ElectrumX and romanz/electrs. -#[repr(i16)] -#[derive(Clone, Copy, PartialEq, Eq)] -enum JsonRpcV2Error { - ParseError = -32700, - InvalidRequest = -32600, - MethodNotFound = -32601, - InvalidParams = -32602, - InternalError = -32603, - BadRequest = 1, - DaemonError = 2, -} - -impl JsonRpcV2Error { - #[inline] - fn into_i16(self) -> i16 { - self as i16 - } -} - fn jsonrpc_code(e: &Error) -> JsonRpcV2Error { match e.kind() { ErrorKind::InvalidParams(_) => JsonRpcV2Error::InvalidParams, @@ -119,22 +100,6 @@ fn jsonrpc_code(e: &Error) -> JsonRpcV2Error { } } -#[inline] -fn json_rpc_error( - input: impl core::fmt::Display, - id: Option<&Value>, - code: JsonRpcV2Error, -) -> Value { - json!({ - "jsonrpc": "2.0", - "id": id.unwrap_or(&Value::Null), - "error": { - "code": code.into_i16(), - "message": format!("{}", input), - }, - }) -} - // TODO: implement caching and delta updates #[trace] fn get_status_hash(txs: Vec<(Txid, Option)>, query: &Query) -> Option { @@ -184,6 +149,9 @@ struct Connection { txs_limit: usize, subscription_limit: usize, max_request_bytes: usize, + max_response_bytes: usize, + write_timeout: Option, + response_budget: Arc, checkpoint_proof_concurrency_limit: usize, #[cfg(feature = "electrum-discovery")] discovery: Option>, @@ -208,6 +176,9 @@ impl Connection { txs_limit: usize, subscription_limit: usize, max_request_bytes: usize, + max_response_bytes: usize, + write_timeout: Option, + response_budget: Arc, checkpoint_proof_concurrency_limit: usize, #[cfg(feature = "electrum-discovery")] discovery: Option>, rpc_logging: RpcLogging, @@ -224,6 +195,9 @@ impl Connection { txs_limit, subscription_limit, max_request_bytes, + max_response_bytes, + write_timeout, + response_budget, checkpoint_proof_concurrency_limit, #[cfg(feature = "electrum-discovery")] discovery, @@ -711,59 +685,60 @@ impl Connection { println!("{}", log); } - fn send_values(&mut self, values: &[Value]) -> Result<()> { - for value in values { - let line = value.to_string() + "\n"; - (&*self.stream) - .write_all(line.as_bytes()) - .chain_err(|| format!("failed to send response ({} bytes)", line.len()))?; - } - Ok(()) - } - #[trace] fn handle_replies(&mut self, receiver: Receiver) -> Result<()> { let empty_params = json!([]); + let stream = Arc::clone(&self.stream); + let stats = Arc::clone(&self.stats); + let write_timeout = self.write_timeout; + let mut response_writer = + ResponseWriter::new(self.max_response_bytes, Arc::clone(&self.response_budget)); loop { let msg = receiver.recv().chain_err(|| "channel closed")?; trace!("RPC {:?}", msg); match msg { Message::Request(line) => { - let reply = match from_str::(&line) { - Ok(Value::Array(arr)) => { - if arr.len() > MAX_ARRAY_BATCH { - bail!( - "Too many elements in batch requests {} max:{}", - arr.len(), - MAX_ARRAY_BATCH - ); - } - let mut result = Vec::with_capacity(arr.len()); - for el in arr { - result.push(self.handle_value(el, &empty_params)); - } - Value::Array(result) - } - Ok(cmd) => self.handle_value(cmd, &empty_params), - Err(err) => { - warn!("[{}] invalid JSON request: {}", self.addr, err); - json_rpc_error("parse error", None, JsonRpcV2Error::ParseError) - } - }; - self.send_values(&[reply])? + let request = from_str::(&line); + drop(line); + if let Err(error) = &request { + warn!("[{}] invalid JSON request: {}", self.addr, error); + } + write_to_client( + &stream, + write_timeout, + &stats.client_write_timeouts, + |writer| { + response_writer.send( + writer, + request, + |command| self.handle_value(command, &empty_params), + |limit| match limit { + ResponseLimit::PerLine => stats.per_line_overflows.inc(), + ResponseLimit::Global => stats.global_overflows.inc(), + }, + ) + }, + )?; } Message::PeriodicUpdate => { let values = self .update_subscriptions() .chain_err(|| "failed to update subscriptions")?; - self.send_values(&values)? + for value in &values { + write_to_client( + &stream, + write_timeout, + &stats.client_write_timeouts, + |writer| response_writer.send_notification(writer, value), + )?; + } } Message::Done => return Ok(()), } } } - fn handle_value(&mut self, cmd: Value, empty_params: &Value) -> Value { + fn handle_value(&mut self, cmd: &Value, empty_params: &Value) -> Value { let start_time = Instant::now(); match ( cmd.get("method"), @@ -894,7 +869,12 @@ impl Connection { Connection::reader_thread(reader, sender, max_request_bytes) }); if let Err(e) = self.handle_replies(receiver) { - if is_disconnect(&e) { + if matches!(e.kind(), ErrorKind::ClientWriteTimeout) { + warn!( + "[{}] client response write timed out; closing connection", + self.addr + ); + } else if is_disconnect(&e) { // client went away mid-exchange (broken pipe / reset) — not actionable debug!("[{}] connection closed by client: {}", self.addr, e); } else { @@ -1130,6 +1110,9 @@ struct Stats { latency: HistogramVec, clients: Gauge, subscriptions: Gauge, + per_line_overflows: Counter, + global_overflows: Counter, + client_write_timeouts: Counter, } impl RPC { @@ -1222,6 +1205,18 @@ impl RPC { "electrum_subscriptions", "# of Electrum subscriptions", )), + per_line_overflows: metrics.counter(MetricOpts::new( + "electrum_per_line_response_overflows", + "Replies rejected for exceeding the per-line response cap", + )), + global_overflows: metrics.counter(MetricOpts::new( + "electrum_global_response_budget_rejections", + "Replies rejected because the global response budget was exhausted", + )), + client_write_timeouts: metrics.counter(MetricOpts::new( + "electrum_client_write_timeouts_total", + "Connections closed because a client response write timed out", + )), }); stats.clients.set(0); stats.subscriptions.set(0); @@ -1256,9 +1251,14 @@ impl RPC { let txs_limit = config.electrum_txs_limit; let subscription_limit = config.electrum_subscription_limit; let max_request_bytes = config.electrum_rpc_max_request_num_bytes; + let max_response_bytes = config.electrum_rpc_max_response_num_bytes; + let write_timeout = config.electrum_rpc_write_timeout; + let response_budget = ResponseBudget::new(config.electrum_rpc_global_response_budget_bytes); let checkpoint_proof_concurrency_limit = config.electrum_checkpoint_proof_concurrency_limit; let conn_max_age = config.electrum_rpc_conn_max_age; + warn_limits(max_request_bytes, max_response_bytes, txs_limit); + RPC { notification: notification.sender(), server: Some(spawn_thread("rpc", move || { @@ -1287,6 +1287,7 @@ impl RPC { while let Some((stream, addr)) = acceptor.receiver().recv().unwrap() { // explicitly scope the shadowed variables for the new thread let query = Arc::clone(&query); + let response_budget = Arc::clone(&response_budget); let stats = Arc::clone(&stats); let garbage_sender = garbage_sender.clone(); let rpc_logging = config.rpc_logging.clone(); @@ -1309,6 +1310,9 @@ impl RPC { txs_limit, subscription_limit, max_request_bytes, + max_response_bytes, + write_timeout, + response_budget, checkpoint_proof_concurrency_limit, #[cfg(feature = "electrum-discovery")] discovery, @@ -1374,6 +1378,7 @@ impl Drop for RPC { #[cfg(test)] mod tests { use super::*; + use std::io::Write; #[test] fn test_hash_ip_with_salt() { diff --git a/src/errors.rs b/src/errors.rs index 8c5acec9f..bd538207b 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -69,6 +69,21 @@ error_chain! { display("Daemon is unavailable: {}", msg) } + PerLineResponseOverflow { + description("Response line exceeds configured cap") + display("response line exceeds max_response_bytes") + } + + ResponseBudgetExhausted { + description("Global response budget exhausted") + display("response budget exhausted, retry later") + } + + ClientWriteTimeout { + description("Client response write timed out") + display("client response write timed out") + } + #[cfg(feature = "electrum-discovery")] ElectrumClient(e: electrum_client::Error) { description("Electrum client error") diff --git a/src/new_index/db.rs b/src/new_index/db.rs index 2ca473dc7..a33ee2cd8 100644 --- a/src/new_index/db.rs +++ b/src/new_index/db.rs @@ -642,7 +642,10 @@ mod tests { cookie: None, electrum_rpc_addr: unused_addr, electrum_rpc_conn_max_age: None, + electrum_rpc_write_timeout: Some(std::time::Duration::from_secs(30)), electrum_rpc_max_request_num_bytes: 1_048_576, + electrum_rpc_max_response_num_bytes: 8_388_608, + electrum_rpc_global_response_budget_bytes: 67_108_864, http_addr: unused_addr, http_socket_file: None, monitoring_addr: unused_addr, @@ -654,6 +657,7 @@ mod tests { cors: None, precache_scripts: None, utxos_limit: 100, + history_scan_limit: 100_000, electrum_txs_limit: 100, electrum_subscription_limit: 10_000, electrum_checkpoint_proof_concurrency_limit: 2, diff --git a/tests/common.rs b/tests/common.rs index 04d198f01..f6010fd8e 100644 --- a/tests/common.rs +++ b/tests/common.rs @@ -106,7 +106,16 @@ impl TestRunner { cookie: None, electrum_rpc_addr: rand_available_addr(), electrum_rpc_conn_max_age: None, + electrum_rpc_write_timeout: Some(std::time::Duration::from_secs(30)), electrum_rpc_max_request_num_bytes: 1_048_576, + #[cfg(not(feature = "liquid"))] + electrum_rpc_max_response_num_bytes: 8_388_608, + #[cfg(feature = "liquid")] + electrum_rpc_max_response_num_bytes: 33_554_432, + #[cfg(not(feature = "liquid"))] + electrum_rpc_global_response_budget_bytes: 67_108_864, + #[cfg(feature = "liquid")] + electrum_rpc_global_response_budget_bytes: 268_435_456, http_addr: rand_available_addr(), http_socket_file: None, // XXX test with socket file or tcp? monitoring_addr: rand_available_addr(), @@ -456,7 +465,13 @@ pub fn init_rest_tester_with( Ok((rest_server, addr, tester)) } pub fn init_electrum_tester() -> Result<(ElectrumRPC, net::SocketAddr, TestRunner)> { - let tester = TestRunner::new()?; + init_electrum_tester_with_config(|_| {}) +} + +pub fn init_electrum_tester_with_config( + configure: impl FnOnce(&mut Config), +) -> Result<(ElectrumRPC, net::SocketAddr, TestRunner)> { + let tester = TestRunner::new_with(configure)?; let addr = tester.config.electrum_rpc_addr; let electrum_server = ElectrumRPC::start( Arc::clone(&tester.config), diff --git a/tests/electrum_response_limits.rs b/tests/electrum_response_limits.rs new file mode 100644 index 000000000..039ba310d --- /dev/null +++ b/tests/electrum_response_limits.rs @@ -0,0 +1,294 @@ +#[allow(dead_code)] +mod common; + +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::{Shutdown, SocketAddr, TcpStream}; +use std::time::{Duration, Instant}; + +use serde_json::{json, Value}; + +struct Client(BufReader); + +impl Client { + fn connect(addr: SocketAddr) -> Self { + let stream = TcpStream::connect(addr).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(15))) + .unwrap(); + stream + .set_write_timeout(Some(Duration::from_secs(15))) + .unwrap(); + Self(BufReader::new(stream)) + } + + fn send(&mut self, request: &Value) { + serde_json::to_writer(self.0.get_mut(), request).unwrap(); + self.0.get_mut().write_all(b"\n").unwrap(); + } + + fn receive(&mut self) -> (Value, usize) { + let mut line = String::new(); + assert!( + self.0.read_line(&mut line).unwrap() > 0, + "connection closed" + ); + assert!(line.ends_with('\n')); + (serde_json::from_str(&line).unwrap(), line.len()) + } + + fn call(&mut self, request: Value) -> (Value, usize) { + self.send(&request); + self.receive() + } +} + +impl Drop for Client { + fn drop(&mut self) { + let _ = self.0.get_ref().shutdown(Shutdown::Both); + } +} + +#[test] +fn unaligned_budget_admits_replies_through_the_exact_line_limit() { + let cap = 100_000; + let banner = "a".repeat(90_000); + let (_server, addr, _tester) = common::init_electrum_tester_with_config(|config| { + config.electrum_rpc_max_response_num_bytes = cap; + config.electrum_rpc_global_response_budget_bytes = cap; + config.electrum_banner = banner.clone(); + }) + .unwrap(); + let mut client = Client::connect(addr); + let (reply, _) = client.call(json!({"id":1,"method":"server.banner"})); + assert_eq!(reply["result"], banner); + + let base = serde_json::to_vec(&json!({"id":"","jsonrpc":"2.0","result":banner})) + .unwrap() + .len() + + 1; + let id = "b".repeat(cap - base); + let (reply, size) = client.call(json!({"id":id,"method":"server.banner"})); + assert_eq!(size, cap); + assert_eq!(reply["id"], id); + assert_eq!(reply["result"], banner); + + let id = format!("{}c", id); + let (reply, size) = client.call(json!({"id":id,"method":"server.banner"})); + assert!(size <= cap); + assert_eq!(reply["id"], id); + assert_eq!(reply["error"]["code"], 1); + + client.0.get_mut().write_all(b"{\n").unwrap(); + let (reply, size) = client.receive(); + assert!(size <= cap); + assert_eq!(reply["id"], Value::Null); + assert_eq!(reply["error"]["code"], -32700); + let (reply, _) = client.call(json!({"id":2,"method":false})); + assert_eq!(reply["id"], 2); + assert_eq!(reply["error"]["code"], -32600); + let (reply, _) = client.call(json!({"id":3,"method":"server.ping"})); + assert_eq!(reply["result"], Value::Null); + assert!(reply.get("error").is_none()); +} + +#[test] +fn response_line_limits_preserve_ids_and_skip_later_commands() { + let cap = 100_000; + let (_server, addr, _tester) = common::init_electrum_tester_with_config(|config| { + config.electrum_rpc_max_response_num_bytes = cap; + config.electrum_rpc_global_response_budget_bytes = usize::MAX; + config.electrum_banner = "a".repeat(80_000); + }) + .unwrap(); + let mut client = Client::connect(addr); + let (single, len) = client.call(json!({"id":1,"method":"server.banner"})); + assert_eq!(single["result"].as_str().unwrap().len(), 80_000); + assert!(len <= cap); + + let hash = "00".repeat(32); + let (batch, len) = client.call(json!([ + {"id":1,"method":"server.banner"}, + {"id":2,"method":"server.banner"}, + {"id":3,"method":"blockchain.scripthash.subscribe","params":[hash]} + ])); + assert!(len <= cap); + assert_eq!(batch[0]["result"], single["result"]); + assert_eq!(batch[1]["id"], 2); + assert_eq!(batch[1]["error"]["code"], 1); + assert_eq!(batch[2]["id"], 3); + assert_eq!(batch[2]["error"]["code"], 1); + let (unsubscribed, _) = + client.call(json!({"id":4,"method":"blockchain.scripthash.unsubscribe","params":[hash]})); + assert_eq!(unsubscribed["result"], false); + let (empty, len) = client.call(json!([])); + assert_eq!(empty, json!([])); + assert_eq!(len, 3); + + let id = "b".repeat(30_000); + let (overflow, len) = client.call(json!({"id":id,"method":"server.banner"})); + assert_eq!(overflow["id"], id); + assert_eq!(overflow["error"]["code"], 1); + assert!(len <= cap); +} + +#[test] +fn response_budget_recovers_when_stalled_batch_writers_time_out() { + let cap = 16 * 1024 * 1024; + let banner_bytes = 4 * 1024 * 1024 - 4096; + let (_server, addr, _tester) = common::init_electrum_tester_with_config(|config| { + config.electrum_rpc_max_response_num_bytes = cap; + config.electrum_rpc_global_response_budget_bytes = cap; + config.electrum_rpc_write_timeout = Some(Duration::from_secs(5)); + config.electrum_rpc_conn_max_age = None; + config.electrum_banner = "a".repeat(banner_bytes); + }) + .unwrap(); + + let mut holders = Vec::new(); + for _ in 0..2 { + // The 8 MiB reply fills the default TCP buffers. Leave the negotiated + // receive window unchanged so reset delivery is reliable on macOS. + let mut client = Client::connect(addr); + client.send(&json!([ + {"id":1,"method":"server.banner"}, + {"id":2,"method":"server.banner"} + ])); + let mut first = [0]; + client.0.read_exact(&mut first).unwrap(); + assert_eq!(first, [b'[']); + holders.push(client); + } + + let mut healthy = Client::connect(addr); + let request = json!({"id":"x".repeat(20 * 1024),"method":"server.ping"}); + let (rejected, _) = healthy.call(request.clone()); + assert_eq!(rejected["error"]["code"], 2); + + // Keep both clients open and unread. Recovery must come from the server's + // write deadline, with connection-age expiry explicitly disabled. + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let (reply, _) = healthy.call(request.clone()); + if reply.get("result").is_some() { + assert_eq!(reply["result"], Value::Null); + break; + } + assert_eq!(reply["error"]["code"], 2); + assert!( + Instant::now() < deadline, + "stalled writers did not release the budget" + ); + std::thread::sleep(Duration::from_millis(20)); + } + let (reply, _) = healthy.call(json!({"id":3,"method":"server.banner"})); + assert_eq!(reply["result"].as_str().unwrap().len(), banner_bytes); + // The first holder's deadline starts first. Recovery can precede the + // second holder's timeout, so keep that client unread while checking reset. + // Buffered bytes may precede the reset on the first connection. + let error = std::io::copy(&mut holders[0].0, &mut std::io::sink()) + .expect_err("a timed-out writer must reset the connection"); + assert_eq!(error.kind(), std::io::ErrorKind::ConnectionReset); +} + +#[test] +fn normal_close_delivers_final_response_before_eof() { + let banner = "a".repeat(256 * 1024); + let (_server, addr, _tester) = common::init_electrum_tester_with_config(|config| { + config.electrum_banner = banner.clone(); + config.electrum_rpc_write_timeout = Some(Duration::from_secs(5)); + }) + .unwrap(); + let mut client = Client::connect(addr); + client.send(&json!({"id":1,"method":"server.banner"})); + // EOF on the request side makes the server close after its final reply. + client.0.get_ref().shutdown(Shutdown::Write).unwrap(); + let (reply, _) = client.receive(); + assert_eq!(reply["result"], banner); + assert_eq!(client.0.read(&mut [0]).unwrap(), 0); +} + +#[test] +fn response_budget_streams_large_ids_and_recovers_after_blocked_writers() { + let cap = 16 * 1024 * 1024; + let banner_bytes = 8 * 1024 * 1024 - 8192; + let (_server, addr, _tester) = common::init_electrum_tester_with_config(|config| { + config.electrum_rpc_max_response_num_bytes = cap; + config.electrum_rpc_global_response_budget_bytes = cap; + config.electrum_banner = "a".repeat(banner_bytes); + }) + .unwrap(); + + let mut holders = Vec::new(); + for id in 0..2 { + let mut client = Client::connect(addr); + socket2::SockRef::from(client.0.get_ref()) + .set_recv_buffer_size(4096) + .unwrap(); + client.send(&json!({"id":id,"method":"server.banner"})); + // Receiving the start proves assembly finished and its reservation is + // held by write_all. The tiny receive window cannot absorb 8 MiB. + let mut first = [0]; + client.0.read_exact(&mut first).unwrap(); + assert_eq!(first, [b'{']); + holders.push(client); + } + + let mut client = Client::connect(addr); + let id = "x".repeat(20 * 1024); + let hash = "00".repeat(32); + let (error, len) = + client.call(json!({"id":id,"method":"blockchain.scripthash.subscribe","params":[hash]})); + assert_eq!(error["id"], id); + assert_eq!(error["error"]["code"], 2); + assert!(len <= cap); + let (unsubscribed, _) = + client.call(json!({"id":4,"method":"blockchain.scripthash.unsubscribe","params":[hash]})); + assert_eq!(unsubscribed["result"], false); + + let (batch, len) = client.call(json!([ + {"id":id,"method":"server.ping"}, + {"id":5,"method":"blockchain.scripthash.subscribe","params":[hash]} + ])); + assert_eq!(batch[0]["id"], id); + assert_eq!(batch[0]["error"]["code"], 2); + assert_eq!(batch[1]["id"], 5); + assert_eq!(batch[1]["error"]["code"], 2); + assert!(len <= cap); + let (unsubscribed, _) = + client.call(json!({"id":6,"method":"blockchain.scripthash.unsubscribe","params":[hash]})); + assert_eq!(unsubscribed["result"], false); + + // Free one holder, leaving 8 MiB available. A banner fits, and the + // reserved error for the next element must fit without another grant. + drop(holders.pop()); + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let (reply, _) = client.call(json!({"id":id,"method":"server.ping"})); + assert_eq!(reply["id"], id); + if reply.get("result").is_some() { + assert_eq!(reply["result"], Value::Null); + break; + } + assert!( + Instant::now() < deadline, + "response budget was not released" + ); + std::thread::sleep(Duration::from_millis(10)); + } + let second_id = "y".repeat(14_000); + let (batch, len) = client.call(json!([ + {"id":7,"method":"server.banner"}, + {"id":second_id,"method":"server.banner"}, + {"id":8,"method":"blockchain.scripthash.subscribe","params":[hash]} + ])); + assert_eq!(batch[0]["result"].as_str().unwrap().len(), banner_bytes); + assert_eq!(batch[1]["id"], second_id); + assert_eq!(batch[1]["error"]["code"], 2); + assert_eq!(batch[2]["id"], 8); + assert_eq!(batch[2]["error"]["code"], 2); + assert!(len <= cap); + let (unsubscribed, _) = + client.call(json!({"id":9,"method":"blockchain.scripthash.unsubscribe","params":[hash]})); + assert_eq!(unsubscribed["result"], false); + drop(holders); +} From 532bd5c0747c1f25f74bfdf80892bf19e999731c Mon Sep 17 00:00:00 2001 From: Oleksandr Fedorenko Date: Mon, 14 Sep 2026 05:35:11 +0300 Subject: [PATCH 19/28] replace checkpoint Merkle proof concurrency cap with a tree cache --- src/config.rs | 26 +- src/new_index/db.rs | 1 + src/new_index/schema.rs | 14 +- src/util/electrum_merkle.rs | 986 ++++++++++++++++++++++++++++++++++-- tests/common.rs | 1 + tests/config.rs | 32 ++ 6 files changed, 1014 insertions(+), 46 deletions(-) diff --git a/src/config.rs b/src/config.rs index ec33366e3..bc82e354e 100644 --- a/src/config.rs +++ b/src/config.rs @@ -79,6 +79,7 @@ pub struct Config { pub electrum_txs_limit: usize, pub electrum_subscription_limit: usize, pub electrum_checkpoint_proof_concurrency_limit: usize, + pub electrum_checkpoint_merkle_cache_mb: usize, pub electrum_banner: String, pub rpc_logging: RpcLogging, pub zmq_addr: Option, @@ -343,8 +344,13 @@ impl Config { ).arg( Arg::with_name("electrum_checkpoint_proof_concurrency_limit") .long("electrum-checkpoint-proof-concurrency-limit") - .help("Maximum number of blockchain.block.header(s) checkpoint Merkle proof builds (triggered by a non-zero cp_height) allowed to run at once, process-wide. Each build hashes every header from genesis up to cp_height, so an unbounded count lets concurrent cheap requests pin every CPU core. Requests past the cap fail immediately rather than queueing. 0 = reject all such requests.") - .default_value("2") + .help("Maximum number of blockchain.block.header(s) checkpoint Merkle proof tree rebuilds (a non-zero cp_height not already cached) allowed to run at once, process-wide. Rebuilds past the cap fail immediately rather than queueing. Cached lookups (the common case once a cp_height has been served once) are never subject to this limit. Default: half the detected CPU cores (min 1). 0 = reject all rebuilds.") + .takes_value(true) + ).arg( + Arg::with_name("electrum_checkpoint_merkle_cache_mb") + .long("electrum-checkpoint-merkle-cache-mb") + .help("Approximate memory budget, in MB, for checkpoint Merkle proof handling: cached trees and the working memory of concurrent rebuilds share this one budget (independently of electrum-checkpoint-proof-concurrency-limit, so a high core count can no longer multiply memory use past it). It is a soft target based on an estimate, not a hard ceiling. Entry size scales with cp_height (up to ~28MB near the chain tip). A cp_height too large to ever fit is served without being cached.") + .default_value("256") .takes_value(true) ).arg( Arg::with_name("electrum_banner") @@ -691,9 +697,21 @@ impl Config { electrum_rpc_global_response_budget_bytes, electrum_txs_limit: value_t_or_exit!(m, "electrum_txs_limit", usize), electrum_subscription_limit: value_t_or_exit!(m, "electrum_subscription_limit", usize), - electrum_checkpoint_proof_concurrency_limit: value_t_or_exit!( + electrum_checkpoint_proof_concurrency_limit: match m + .value_of("electrum_checkpoint_proof_concurrency_limit") + { + Some(v) => v.parse::().unwrap_or_else(|_| { + clap::Error::value_validation_auto(format!( + "The argument '{}' isn't a valid value", + v + )) + .exit() + }), + None => (num_cpus::get() / 2).max(1), + }, + electrum_checkpoint_merkle_cache_mb: value_t_or_exit!( m, - "electrum_checkpoint_proof_concurrency_limit", + "electrum_checkpoint_merkle_cache_mb", usize ), electrum_banner, diff --git a/src/new_index/db.rs b/src/new_index/db.rs index a33ee2cd8..e43d7a5f2 100644 --- a/src/new_index/db.rs +++ b/src/new_index/db.rs @@ -661,6 +661,7 @@ mod tests { electrum_txs_limit: 100, electrum_subscription_limit: 10_000, electrum_checkpoint_proof_concurrency_limit: 2, + electrum_checkpoint_merkle_cache_mb: 256, electrum_banner: String::new(), rpc_logging: RpcLogging::default(), zmq_addr: None, diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index a357848d9..7bb52f61f 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -27,8 +27,8 @@ use crate::daemon::Daemon; use crate::errors::*; use crate::metrics::{Gauge, HistogramOpts, HistogramTimer, HistogramVec, MetricOpts, Metrics}; use crate::util::{ - bincode, full_hash, has_prevout, is_spendable, BlockHeaderMeta, BlockId, BlockMeta, - BlockStatus, Bytes, HeaderEntry, HeaderList, ScriptToAddr, + bincode, electrum_merkle::CheckpointMerkleCache, full_hash, has_prevout, is_spendable, + BlockHeaderMeta, BlockId, BlockMeta, BlockStatus, Bytes, HeaderEntry, HeaderList, ScriptToAddr, }; use crate::new_index::db::{DBFlush, DBRow, ReverseScanIterator, ScanIterator, DB}; @@ -293,6 +293,7 @@ pub struct ChainQuery { duration: HistogramVec, network: Network, history_scan_limit: usize, + checkpoint_merkle_cache: CheckpointMerkleCache, } // TODO: &[Block] should be an iterator / a queue. @@ -640,6 +641,11 @@ impl ChainQuery { HistogramOpts::new("query_duration", "Index query duration (in seconds)"), &["name"], ), + checkpoint_merkle_cache: CheckpointMerkleCache::new( + config + .electrum_checkpoint_merkle_cache_mb + .saturating_mul(1024 * 1024), + ), } } @@ -651,6 +657,10 @@ impl ChainQuery { &self.store } + pub fn checkpoint_merkle_cache(&self) -> &CheckpointMerkleCache { + &self.checkpoint_merkle_cache + } + fn start_timer(&self, name: &str) -> HistogramTimer { self.duration.with_label_values(&[name]).start_timer() } diff --git a/src/util/electrum_merkle.rs b/src/util/electrum_merkle.rs index 7bf813afd..716a61e56 100644 --- a/src/util/electrum_merkle.rs +++ b/src/util/electrum_merkle.rs @@ -2,38 +2,287 @@ use crate::chain::{BlockHash, Txid}; use crate::errors::*; use crate::new_index::ChainQuery; use bitcoin::hashes::{sha256d::Hash as Sha256dHash, Hash}; +use std::collections::HashMap; use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; use electrs_macros::trace; -static INFLIGHT_CHECKPOINT_PROOFS: AtomicUsize = AtomicUsize::new(0); +const MAX_CACHE_ENTRIES: usize = 256; -struct InflightCheckpointProofGuard; +struct CachedLevels { + cp_hash: BlockHash, + levels: Vec>, + size_bytes: usize, +} + +impl CachedLevels { + fn new(cp_hash: BlockHash, levels: Vec>) -> Self { + let size_bytes = levels.iter().map(|level| level.len()).sum::() + * std::mem::size_of::(); + Self { + cp_hash, + levels, + size_bytes, + } + } +} + +struct RebuildPermit<'a> { + counter: &'a AtomicUsize, +} -impl InflightCheckpointProofGuard { - fn acquire(limit: usize) -> Result { - let mut current = INFLIGHT_CHECKPOINT_PROOFS.load(Ordering::Relaxed); +impl<'a> RebuildPermit<'a> { + fn acquire(counter: &'a AtomicUsize, limit: usize) -> Result { + let mut current = counter.load(Ordering::Relaxed); loop { if current >= limit { - bail!("too many concurrent checkpoint merkle proof requests, try again later"); + bail!("too many concurrent checkpoint merkle proof rebuilds, try again later"); } - match INFLIGHT_CHECKPOINT_PROOFS.compare_exchange_weak( + match counter.compare_exchange_weak( current, current + 1, Ordering::AcqRel, Ordering::Relaxed, ) { - Ok(_) => return Ok(Self), + Ok(_) => return Ok(Self { counter }), + Err(observed) => current = observed, + } + } + } +} + +impl<'a> Drop for RebuildPermit<'a> { + fn drop(&mut self) { + self.counter.fetch_sub(1, Ordering::AcqRel); + } +} + +struct MemoryPermit<'a> { + counter: &'a AtomicUsize, + weight_bytes: usize, +} + +impl<'a> MemoryPermit<'a> { + fn acquire(counter: &'a AtomicUsize, limit_bytes: usize, weight_bytes: usize) -> Result { + let mut current = counter.load(Ordering::Relaxed); + loop { + if current > 0 && current.saturating_add(weight_bytes) > limit_bytes { + bail!("checkpoint merkle proof rebuild would exceed the configured memory budget, try again later"); + } + match counter.compare_exchange_weak( + current, + current + weight_bytes, + Ordering::AcqRel, + Ordering::Relaxed, + ) { + Ok(_) => { + return Ok(Self { + counter, + weight_bytes, + }) + } Err(observed) => current = observed, } } } } -impl Drop for InflightCheckpointProofGuard { +impl<'a> Drop for MemoryPermit<'a> { fn drop(&mut self) { - INFLIGHT_CHECKPOINT_PROOFS.fetch_sub(1, Ordering::AcqRel); + self.counter.fetch_sub(self.weight_bytes, Ordering::AcqRel); + } +} + +fn estimated_peak_build_bytes(cp_height: usize) -> usize { + (cp_height + 1) + .saturating_mul(std::mem::size_of::()) + .saturating_mul(3) +} + +type BuildSlot = Mutex>>; + +pub struct CheckpointMerkleCache { + entries: Mutex)>>, + build_locks: Mutex>>, + inflight_rebuilds: AtomicUsize, + inflight_build_bytes: AtomicUsize, + capacity_bytes: usize, +} + +impl CheckpointMerkleCache { + pub fn new(capacity_bytes: usize) -> Self { + Self { + entries: Mutex::new(Vec::new()), + build_locks: Mutex::new(HashMap::new()), + inflight_rebuilds: AtomicUsize::new(0), + inflight_build_bytes: AtomicUsize::new(0), + capacity_bytes, + } + } + + fn get_valid(&self, cp_height: usize, cp_hash: BlockHash) -> Option> { + let entries = self.entries.lock().unwrap(); + entries.iter().find_map(|(h, cached)| { + (*h == cp_height && cached.cp_hash == cp_hash).then(|| cached.clone()) + }) + } + + fn insert(&self, cp_height: usize, best_height: usize, cached: Arc) { + if cached.size_bytes > self.capacity_bytes { + return; + } + + let mut entries = self.entries.lock().unwrap(); + entries.retain(|(h, _)| *h != cp_height && *h <= best_height); + + let mut total_bytes: usize = entries.iter().map(|(_, c)| c.size_bytes).sum(); + while total_bytes + cached.size_bytes > self.capacity_bytes + || entries.len() >= MAX_CACHE_ENTRIES + { + match entries.iter().enumerate().min_by_key(|(_, (h, _))| *h) { + Some((pos, (min_height, _))) if cp_height > *min_height => { + total_bytes -= entries[pos].1.size_bytes; + entries.remove(pos); + } + _ => return, + } + } + + entries.push((cp_height, cached)); + } + + fn build_lock_for(&self, cp_height: usize) -> Arc { + let mut locks = self.build_locks.lock().unwrap(); + locks + .entry(cp_height) + .or_insert_with(|| Arc::new(Mutex::new(None))) + .clone() + } + + fn release_build_lock(&self, cp_height: usize, lock: &Arc) { + let mut locks = self.build_locks.lock().unwrap(); + if let Some(existing) = locks.get(&cp_height) { + if Arc::ptr_eq(existing, lock) && Arc::strong_count(existing) <= 2 { + locks.remove(&cp_height); + } + } + } + + fn get_or_build( + &self, + cp_height: usize, + best_height: usize, + concurrency_limit: usize, + snapshot_cp_hash_and_sibling: impl Fn() -> Result<(BlockHash, Sha256dHash)>, + build: impl FnOnce() -> Result<(BlockHash, Vec>, Sha256dHash)>, + ) -> Result<(Arc, Sha256dHash)> { + let (cp_hash, sibling) = snapshot_cp_hash_and_sibling()?; + if let Some(cached) = self.get_valid(cp_height, cp_hash) { + return Ok((cached, sibling)); + } + + let build_lock = self.build_lock_for(cp_height); + let result = self.build_under_lock( + cp_height, + best_height, + cp_hash, + sibling, + concurrency_limit, + &build_lock, + build, + ); + self.release_build_lock(cp_height, &build_lock); + result + } + + fn build_under_lock( + &self, + cp_height: usize, + best_height: usize, + cp_hash: BlockHash, + sibling: Sha256dHash, + concurrency_limit: usize, + build_lock: &BuildSlot, + build: impl FnOnce() -> Result<(BlockHash, Vec>, Sha256dHash)>, + ) -> Result<(Arc, Sha256dHash)> { + let mut last_built = build_lock.lock().unwrap(); + + if let Some(cached) = self.get_valid(cp_height, cp_hash) { + return Ok((cached, sibling)); + } + if let Some(cached) = last_built.as_ref().filter(|c| c.cp_hash == cp_hash) { + return Ok((cached.clone(), sibling)); + } + + let _permit = RebuildPermit::acquire(&self.inflight_rebuilds, concurrency_limit)?; + let entries = self.entries.lock().unwrap(); + let resident_bytes: usize = entries.iter().map(|(_, cached)| cached.size_bytes).sum(); + let _memory_permit = MemoryPermit::acquire( + &self.inflight_build_bytes, + self.capacity_bytes.saturating_sub(resident_bytes), + estimated_peak_build_bytes(cp_height), + )?; + drop(entries); + let (cp_hash, levels, sibling) = build()?; + let cached = Arc::new(CachedLevels::new(cp_hash, levels)); + self.insert(cp_height, best_height, cached.clone()); + *last_built = Some(cached.clone()); + Ok((cached, sibling)) + } +} + +fn merklize(left: Sha256dHash, right: Sha256dHash) -> Sha256dHash { + let data = [&left[..], &right[..]].concat(); + Sha256dHash::hash(&data) +} + +fn build_levels_above_leaves(leaves: &[Sha256dHash]) -> Result>> { + ensure!(!leaves.is_empty(), "missing block headers"); + let mut current: Vec = leaves.to_vec(); + + let mut levels = Vec::new(); + while current.len() > 1 { + if current.len() % 2 != 0 { + let last = *current.last().unwrap(); + current.reserve_exact(1); + current.push(last); + } + current = current + .chunks(2) + .map(|pair| merklize(pair[0], pair[1])) + .collect(); + levels.push(current.clone()); + } + Ok(levels) +} + +fn extract_branch_and_root( + levels_above_leaves: &[Vec], + sibling_leaf: Sha256dHash, + mut index: usize, +) -> Result<(Vec, Sha256dHash)> { + let mut merkle = vec![sibling_leaf]; + index /= 2; + + for level in &levels_above_leaves[..levels_above_leaves.len().saturating_sub(1)] { + let len = level.len(); + let sibling_index = if index % 2 == 0 { index + 1 } else { index - 1 }; + let sibling = if sibling_index < len { + level[sibling_index] + } else { + level[len - 1] + }; + merkle.push(sibling); + index /= 2; } + + let root = *levels_above_leaves + .last() + .chain_err(|| "empty checkpoint merkle tree")? + .last() + .unwrap(); + Ok((merkle, root)) } #[trace] @@ -75,18 +324,66 @@ pub fn get_header_merkle_proof( ); } - let _guard = InflightCheckpointProofGuard::acquire(checkpoint_proof_concurrency_limit)?; + if cp_height == 0 { + let headers = chain.store().headers(); + let root = Sha256dHash::from( + *headers + .header_by_height(0) + .chain_err(|| "missing block header at height 0")? + .hash(), + ); + return Ok((vec![], root)); + } + + let sibling_index = if height % 2 == 0 { height + 1 } else { height - 1 }; + let sibling_height = sibling_index.min(cp_height); + + let snapshot_cp_hash_and_sibling = || -> Result<(BlockHash, Sha256dHash)> { + let headers = chain.store().headers(); + let cp_hash = *headers + .header_by_height(cp_height) + .chain_err(|| format!("missing block header at height {}", cp_height))? + .hash(); + let sibling = Sha256dHash::from( + *headers + .header_by_height(sibling_height) + .chain_err(|| format!("missing block header at height {}", sibling_height))? + .hash(), + ); + Ok((cp_hash, sibling)) + }; + + let build = || -> Result<(BlockHash, Vec>, Sha256dHash)> { + let headers = chain.store().headers(); + let cp_hash = *headers + .header_by_height(cp_height) + .chain_err(|| format!("missing block header at height {}", cp_height))? + .hash(); + let leaves: Vec = (0..=cp_height) + .map(|h| { + headers + .header_by_height(h) + .map(|entry| Sha256dHash::from(*entry.hash())) + }) + .collect::>>() + .chain_err(|| "missing block headers")?; + drop(headers); + + let sibling = leaves[sibling_height]; + let levels = build_levels_above_leaves(&leaves)?; + Ok((cp_hash, levels, sibling)) + }; - let heights: Vec = (0..=cp_height).collect(); - let header_hashes: Vec = heights - .into_iter() - .map(|height| chain.hash_by_height(height)) - .collect::>>() - .chain_err(|| "missing block headers")?; + let (cached, sibling) = chain.checkpoint_merkle_cache().get_or_build(cp_height, + best_height, + checkpoint_proof_concurrency_limit, + snapshot_cp_hash_and_sibling, + build, + )?; - let header_hashes = header_hashes.into_iter().map(Sha256dHash::from).collect(); - Ok(create_merkle_branch_and_root(header_hashes, height)) + extract_branch_and_root(&cached.levels, sibling, height) } + #[trace] pub fn get_id_from_pos( chain: &ChainQuery, @@ -116,11 +413,6 @@ pub fn get_id_from_pos( Ok((txid, branch)) } -fn merklize(left: Sha256dHash, right: Sha256dHash) -> Sha256dHash { - let data = [&left[..], &right[..]].concat(); - Sha256dHash::hash(&data) -} - fn create_merkle_branch_and_root( mut hashes: Vec, mut index: usize, @@ -145,30 +437,644 @@ fn create_merkle_branch_and_root( #[cfg(test)] mod tests { use super::*; + use std::sync::Barrier; + use std::thread; + + const GENEROUS_TEST_CACHE_BYTES: usize = 1024 * 1024; + + fn leaves(n: usize) -> Vec { + (0..n).map(|i| Sha256dHash::hash(&[i as u8])).collect() + } + + fn sibling_leaf(leaves: &[Sha256dHash], cp_height: usize, height: usize) -> Sha256dHash { + let sibling_index = if height % 2 == 0 { height + 1 } else { height - 1 }; + leaves[sibling_index.min(cp_height)] + } + + #[test] + fn incremental_levels_match_naive_algorithm_for_every_height() { + for n in 1..=17usize { + let leaves = leaves(n); + let cp_height = n - 1; + + for height in 0..=cp_height { + let (expected_branch, expected_root) = + create_merkle_branch_and_root(leaves.clone(), height); + + let (branch, root) = if cp_height == 0 { + (vec![], leaves[0]) + } else { + let levels = build_levels_above_leaves(&leaves).unwrap(); + let sibling = sibling_leaf(&leaves, cp_height, height); + extract_branch_and_root(&levels, sibling, height).unwrap() + }; + + assert_eq!( + branch, expected_branch, + "branch mismatch for n={} height={}", + n, height + ); + assert_eq!( + root, expected_root, + "root mismatch for n={} height={}", + n, height + ); + } + } + } - // Single test so the shared static isn't touched by other tests running - // concurrently in the same process. #[test] - fn checkpoint_proof_guard_caps_concurrency_and_releases_on_drop() { + fn cache_hit_serves_different_heights_without_rebuilding() { + let leaves = leaves(9); + let cp_height = 8; + let cp_hash = BlockHash::hash(&[7u8]); + + let cache = CheckpointMerkleCache::new(GENEROUS_TEST_CACHE_BYTES); + let build_count = AtomicUsize::new(0); + + for height in [0usize, 3, 8] { + let sibling = sibling_leaf(&leaves, cp_height, height); + let (cached, _) = cache + .get_or_build(cp_height, usize::MAX, + 1, + || Ok((cp_hash, sibling)), + || { + build_count.fetch_add(1, Ordering::SeqCst); + Ok((cp_hash, build_levels_above_leaves(&leaves)?, sibling)) + }, + ) + .unwrap(); + let (branch, root) = extract_branch_and_root(&cached.levels, sibling, height).unwrap(); + let (expected_branch, expected_root) = + create_merkle_branch_and_root(leaves.clone(), height); + assert_eq!(branch, expected_branch); + assert_eq!(root, expected_root); + } + + assert_eq!( + build_count.load(Ordering::SeqCst), + 1, + "second and third lookups should hit the cache, not rebuild" + ); + } + + #[test] + fn stale_entry_is_rebuilt_after_checkpoint_hash_changes() { + let leaves = leaves(5); + let cp_height = 4; + let sibling = sibling_leaf(&leaves, cp_height, 0); + let old_hash = BlockHash::hash(&[1u8]); + let new_hash = BlockHash::hash(&[2u8]); + + let cache = CheckpointMerkleCache::new(GENEROUS_TEST_CACHE_BYTES); + let build_count = AtomicUsize::new(0); + + cache + .get_or_build(cp_height, usize::MAX, 4, || Ok((old_hash, sibling)), || { + build_count.fetch_add(1, Ordering::SeqCst); + Ok((old_hash, build_levels_above_leaves(&leaves)?, sibling)) + }) + .unwrap(); + cache + .get_or_build(cp_height, usize::MAX, 4, || Ok((old_hash, sibling)), || { + build_count.fetch_add(1, Ordering::SeqCst); + Ok((old_hash, build_levels_above_leaves(&leaves)?, sibling)) + }) + .unwrap(); + assert_eq!(build_count.load(Ordering::SeqCst), 1); + + cache + .get_or_build(cp_height, usize::MAX, 4, || Ok((new_hash, sibling)), || { + build_count.fetch_add(1, Ordering::SeqCst); + Ok((new_hash, build_levels_above_leaves(&leaves)?, sibling)) + }) + .unwrap(); + assert_eq!( + build_count.load(Ordering::SeqCst), + 2, + "a changed checkpoint hash must trigger a rebuild" + ); + } + + #[test] + fn stale_entry_returns_correct_branch_and_root_after_checkpoint_hash_changes() { + let old_leaves = leaves(5); + let new_leaves: Vec = (100..105).map(|i| Sha256dHash::hash(&[i as u8])).collect(); + let cp_height = 4; + let height = 1; + let old_sibling = sibling_leaf(&old_leaves, cp_height, height); + let new_sibling = sibling_leaf(&new_leaves, cp_height, height); + let old_hash = BlockHash::hash(&[1u8]); + let new_hash = BlockHash::hash(&[2u8]); + + let cache = CheckpointMerkleCache::new(GENEROUS_TEST_CACHE_BYTES); + + let (old_cached, old_ret_sibling) = cache + .get_or_build(cp_height, usize::MAX, 4, || Ok((old_hash, old_sibling)), || { + Ok((old_hash, build_levels_above_leaves(&old_leaves)?, old_sibling)) + }) + .unwrap(); + let (old_branch, old_root) = + extract_branch_and_root(&old_cached.levels, old_ret_sibling, height).unwrap(); + let (expected_old_branch, expected_old_root) = + create_merkle_branch_and_root(old_leaves.clone(), height); + assert_eq!(old_branch, expected_old_branch); + assert_eq!(old_root, expected_old_root); + + let (new_cached, new_ret_sibling) = cache + .get_or_build(cp_height, usize::MAX, 4, || Ok((new_hash, new_sibling)), || { + Ok((new_hash, build_levels_above_leaves(&new_leaves)?, new_sibling)) + }) + .unwrap(); + let (new_branch, new_root) = + extract_branch_and_root(&new_cached.levels, new_ret_sibling, height).unwrap(); + let (expected_new_branch, expected_new_root) = + create_merkle_branch_and_root(new_leaves.clone(), height); + assert_eq!(new_branch, expected_new_branch); + assert_eq!(new_root, expected_new_root); + assert_ne!( + new_root, old_root, + "a rebuild after a checkpoint hash change must reflect the new chain state, not the stale cached one" + ); + } + + #[test] + fn insert_evicts_lowest_height_for_a_higher_value() { + let entry_leaves = leaves(2); + let entry_bytes = build_levels_above_leaves(&entry_leaves).unwrap()[0].len() + * std::mem::size_of::(); + let cache = CheckpointMerkleCache::new(5 * entry_bytes); + let leaf = entry_leaves[0]; + + for cp_height in [10usize, 20, 30, 40, 50] { + let hash = BlockHash::hash(&[cp_height as u8]); + cache + .get_or_build(cp_height, usize::MAX, + 4, + || Ok((hash, leaf)), + || Ok((hash, build_levels_above_leaves(&entry_leaves)?, leaf)), + ) + .unwrap(); + } + assert_eq!(cache.entries.lock().unwrap().len(), 5); + + let hash60 = BlockHash::hash(&[60u8]); + cache + .get_or_build(60, usize::MAX, + 4, + || Ok((hash60, leaf)), + || Ok((hash60, build_levels_above_leaves(&entry_leaves)?, leaf)), + ) + .unwrap(); + + let heights: Vec = cache + .entries + .lock() + .unwrap() + .iter() + .map(|(h, _)| *h) + .collect(); + assert_eq!(heights.len(), 5); + assert!( + !heights.contains(&10), + "lowest cp_height should be evicted, got {:?}", + heights + ); + assert!(heights.contains(&60)); + } + + #[test] + fn insert_skips_values_below_the_cached_minimum() { + let entry_leaves = leaves(2); + let entry_bytes = build_levels_above_leaves(&entry_leaves).unwrap()[0].len() + * std::mem::size_of::(); + let cache = CheckpointMerkleCache::new(5 * entry_bytes); + let leaf = entry_leaves[0]; + + for cp_height in [10usize, 20, 30, 40, 50] { + let hash = BlockHash::hash(&[cp_height as u8]); + cache + .get_or_build(cp_height, usize::MAX, + 4, + || Ok((hash, leaf)), + || Ok((hash, build_levels_above_leaves(&entry_leaves)?, leaf)), + ) + .unwrap(); + } + assert_eq!(cache.entries.lock().unwrap().len(), 5); + + let hash5 = BlockHash::hash(&[5u8]); + cache + .get_or_build(5, usize::MAX, + 4, + || Ok((hash5, leaf)), + || Ok((hash5, build_levels_above_leaves(&entry_leaves)?, leaf)), + ) + .unwrap(); + + let heights: Vec = cache + .entries + .lock() + .unwrap() + .iter() + .map(|(h, _)| *h) + .collect(); + assert_eq!(heights.len(), 5); + assert!( + heights.contains(&10), + "a lower cp_height must not displace an existing higher entry, got {:?}", + heights + ); + assert!(!heights.contains(&5)); + } + + #[test] + fn insert_caps_entry_count_even_under_a_generous_byte_budget() { + let entry_leaves = leaves(2); + let entry_bytes = build_levels_above_leaves(&entry_leaves).unwrap()[0].len() + * std::mem::size_of::(); + let cache = CheckpointMerkleCache::new((MAX_CACHE_ENTRIES + 100) * entry_bytes); + let leaf = entry_leaves[0]; + let hash = BlockHash::hash(&[7u8]); + + for cp_height in 0..MAX_CACHE_ENTRIES + 50 { + cache + .get_or_build(cp_height, usize::MAX, + 4, + || Ok((hash, leaf)), + || Ok((hash, build_levels_above_leaves(&entry_leaves)?, leaf)), + ) + .unwrap(); + } + + let len = cache.entries.lock().unwrap().len(); + assert!( + len <= MAX_CACHE_ENTRIES, + "entry count exceeded the cap: {}", + len + ); + } + + #[test] + fn insert_prunes_entries_orphaned_by_a_shorter_reorg() { + let entry_leaves = leaves(2); + let entry_bytes = build_levels_above_leaves(&entry_leaves).unwrap()[0].len() + * std::mem::size_of::(); + let leaf = entry_leaves[0]; + let cache = CheckpointMerkleCache::new(entry_bytes); + + let old_height = 100; + let old_hash = BlockHash::hash(&[1u8]); + cache + .get_or_build(old_height, old_height, 4, || Ok((old_hash, leaf)), || { + Ok((old_hash, build_levels_above_leaves(&entry_leaves)?, leaf)) + }) + .unwrap(); + assert_eq!(cache.entries.lock().unwrap().len(), 1); + + let new_height = old_height - 1; + let new_hash = BlockHash::hash(&[2u8]); + let build_count = AtomicUsize::new(0); + + cache + .get_or_build(new_height, new_height, 4, || Ok((new_hash, leaf)), || { + build_count.fetch_add(1, Ordering::SeqCst); + Ok((new_hash, build_levels_above_leaves(&entry_leaves)?, leaf)) + }) + .unwrap(); + assert_eq!(build_count.load(Ordering::SeqCst), 1); + + cache + .get_or_build(new_height, new_height, 4, || Ok((new_hash, leaf)), || { + build_count.fetch_add(1, Ordering::SeqCst); + panic!("must serve from cache instead of rebuilding") + }) + .unwrap(); + assert_eq!( + build_count.load(Ordering::SeqCst), + 1, + "the post-reorg checkpoint must be cached once built, not rebuilt on every request \ + just because the old, now-unreachable height still occupied the budget" + ); + + let heights: Vec = cache + .entries + .lock() + .unwrap() + .iter() + .map(|(h, _)| *h) + .collect(); + assert_eq!(heights, vec![new_height]); + } + + #[test] + fn concurrent_rebuilds_of_same_cp_height_run_build_once() { + let leaves = Arc::new(leaves(9)); + let cp_height = 8; + let cp_hash = BlockHash::hash(&[9u8]); + let sibling = sibling_leaf(&leaves, cp_height, 0); + + let cache = Arc::new(CheckpointMerkleCache::new(GENEROUS_TEST_CACHE_BYTES)); + let build_count = Arc::new(AtomicUsize::new(0)); + let barrier = Arc::new(Barrier::new(8)); + + let handles: Vec<_> = (0..8) + .map(|_| { + let cache = cache.clone(); + let build_count = build_count.clone(); + let barrier = barrier.clone(); + let leaves = leaves.clone(); + thread::spawn(move || { + barrier.wait(); + cache + .get_or_build(cp_height, usize::MAX, + 8, + || Ok((cp_hash, sibling)), + || { + build_count.fetch_add(1, Ordering::SeqCst); + thread::sleep(std::time::Duration::from_millis(20)); + Ok((cp_hash, build_levels_above_leaves(&leaves)?, sibling)) + }, + ) + .unwrap(); + }) + }) + .collect(); + + for h in handles { + h.join().unwrap(); + } + + assert_eq!( + build_count.load(Ordering::SeqCst), + 1, + "8 concurrent requests for the same cp_height must build exactly once" + ); + } + + #[test] + fn concurrent_requests_for_an_uncacheable_height_still_build_once() { + let leaves = Arc::new(leaves(9)); + let cp_height = 8; + let cp_hash = BlockHash::hash(&[9u8]); + let sibling = sibling_leaf(&leaves, cp_height, 0); + + let cache = Arc::new(CheckpointMerkleCache::new(0)); + let build_count = Arc::new(AtomicUsize::new(0)); + let barrier = Arc::new(Barrier::new(8)); + let _keep_build_slot_alive = cache.build_lock_for(cp_height); + + let handles: Vec<_> = (0..8) + .map(|_| { + let cache = cache.clone(); + let build_count = build_count.clone(); + let barrier = barrier.clone(); + let leaves = leaves.clone(); + thread::spawn(move || { + barrier.wait(); + cache + .get_or_build(cp_height, usize::MAX, + 8, + || Ok((cp_hash, sibling)), + || { + build_count.fetch_add(1, Ordering::SeqCst); + thread::sleep(std::time::Duration::from_millis(20)); + Ok((cp_hash, build_levels_above_leaves(&leaves)?, sibling)) + }, + ) + .unwrap(); + }) + }) + .collect(); + + for h in handles { + h.join().unwrap(); + } + + assert_eq!( + build_count.load(Ordering::SeqCst), + 1, + "8 concurrent requests for the same uncacheable cp_height must still build exactly once" + ); + } + + #[test] + fn rebuild_permit_caps_concurrency_and_releases_on_drop() { const LIMIT: usize = 2; - let mut guards = Vec::new(); + let counter = AtomicUsize::new(0); + let mut permits = Vec::new(); for _ in 0..LIMIT { - guards.push(InflightCheckpointProofGuard::acquire(LIMIT).unwrap()); + permits.push(RebuildPermit::acquire(&counter, LIMIT).unwrap()); } - // All permits are taken: the next caller must be rejected outright - // instead of blocking or panicking. - assert!(InflightCheckpointProofGuard::acquire(LIMIT).is_err()); - - // Releasing one permit (drop, e.g. on early return via `?`) must let - // the next acquire through. - guards.pop(); - let extra = InflightCheckpointProofGuard::acquire(LIMIT).unwrap(); + assert!(RebuildPermit::acquire(&counter, LIMIT).is_err()); - // Back at the cap: still no free permits. - assert!(InflightCheckpointProofGuard::acquire(LIMIT).is_err()); + permits.pop(); + let extra = RebuildPermit::acquire(&counter, LIMIT).unwrap(); + assert!(RebuildPermit::acquire(&counter, LIMIT).is_err()); drop(extra); - drop(guards); + drop(permits); + } + + #[test] + fn cache_hits_bypass_the_rebuild_permit_entirely() { + let leaves = leaves(3); + let cp_height = 2; + let sibling = sibling_leaf(&leaves, cp_height, 0); + let cp_hash = BlockHash::hash(&[3u8]); + + let cache = CheckpointMerkleCache::new(GENEROUS_TEST_CACHE_BYTES); + cache + .get_or_build(cp_height, usize::MAX, 1, || Ok((cp_hash, sibling)), || { + Ok((cp_hash, build_levels_above_leaves(&leaves)?, sibling)) + }) + .unwrap(); + + cache + .get_or_build(cp_height, usize::MAX, 0, || Ok((cp_hash, sibling)), || { + panic!("must not rebuild on a cache hit") + }) + .unwrap(); + } + + #[test] + fn cache_hit_succeeds_while_all_rebuild_permits_are_held() { + let leaves = leaves(3); + let cp_height = 2; + let sibling = sibling_leaf(&leaves, cp_height, 0); + let cp_hash = BlockHash::hash(&[3u8]); + + let cache = Arc::new(CheckpointMerkleCache::new(GENEROUS_TEST_CACHE_BYTES)); + cache + .get_or_build(cp_height, usize::MAX, 1, || Ok((cp_hash, sibling)), || { + Ok((cp_hash, build_levels_above_leaves(&leaves)?, sibling)) + }) + .unwrap(); + + const HOLDERS: usize = 2; + let inside_build = Arc::new(Barrier::new(HOLDERS + 1)); + let release_build = Arc::new(Barrier::new(HOLDERS + 1)); + + let handles: Vec<_> = (0..HOLDERS) + .map(|i| { + let cache = cache.clone(); + let inside_build = inside_build.clone(); + let release_build = release_build.clone(); + thread::spawn(move || { + let miss_height = 1_000 + i; + let hash = BlockHash::hash(&[100 + i as u8]); + let leaf = Sha256dHash::hash(&[0u8]); + cache + .get_or_build(miss_height, usize::MAX, + HOLDERS, + || Ok((hash, leaf)), + || { + inside_build.wait(); + release_build.wait(); + Ok((hash, build_levels_above_leaves(&[leaf])?, leaf)) + }, + ) + .unwrap(); + }) + }) + .collect(); + + inside_build.wait(); + + cache + .get_or_build(cp_height, usize::MAX, 0, || Ok((cp_hash, sibling)), || { + panic!("must not rebuild on a cache hit") + }) + .unwrap(); + + release_build.wait(); + for h in handles { + h.join().unwrap(); + } + } + + #[test] + fn memory_permit_caps_concurrent_rebuild_bytes_independently_of_concurrency_limit() { + const HOLDERS: usize = 2; + let holder_heights = [1_000usize, 2_000usize]; + let capacity_bytes: usize = holder_heights + .iter() + .map(|h| estimated_peak_build_bytes(*h)) + .sum(); + + let cache = Arc::new(CheckpointMerkleCache::new(capacity_bytes)); + let inside_build = Arc::new(Barrier::new(HOLDERS + 1)); + let release_build = Arc::new(Barrier::new(HOLDERS + 1)); + + let handles: Vec<_> = holder_heights + .iter() + .map(|&miss_height| { + let cache = cache.clone(); + let inside_build = inside_build.clone(); + let release_build = release_build.clone(); + thread::spawn(move || { + let hash = BlockHash::hash(&[miss_height as u8]); + let leaf = Sha256dHash::hash(&[0u8]); + cache + .get_or_build(miss_height, usize::MAX, + 100, + || Ok((hash, leaf)), + || { + inside_build.wait(); + release_build.wait(); + Ok((hash, build_levels_above_leaves(&[leaf])?, leaf)) + }, + ) + .unwrap(); + }) + }) + .collect(); + + inside_build.wait(); + + let result = cache.get_or_build(3_000, usize::MAX, 100, || Ok((BlockHash::hash(&[9u8]), Sha256dHash::hash(&[0u8]))), || { + panic!("must not rebuild once the memory budget is fully held") + }); + assert!( + result.is_err(), + "a third concurrent rebuild must be rejected once the memory budget is exhausted, even with a generous concurrency_limit" + ); + + release_build.wait(); + for h in handles { + h.join().unwrap(); + } + } + + #[test] + fn memory_permit_budget_accounts_for_already_resident_cache_bytes() { + let resident_leaves = leaves(2); + let resident_bytes = build_levels_above_leaves(&resident_leaves).unwrap()[0].len() + * std::mem::size_of::(); + let resident_height = 500usize; + let resident_hash = BlockHash::hash(&[7u8]); + let resident_sibling = resident_leaves[0]; + + let holder_height = 1_000usize; + let extra_height = 2_000usize; + let capacity_bytes = resident_bytes + + estimated_peak_build_bytes(holder_height) + + estimated_peak_build_bytes(extra_height) + - 1; + + let cache = Arc::new(CheckpointMerkleCache::new(capacity_bytes)); + cache + .get_or_build(resident_height, usize::MAX, + 1, + || Ok((resident_hash, resident_sibling)), + || { + Ok(( + resident_hash, + build_levels_above_leaves(&resident_leaves)?, + resident_sibling, + )) + }, + ) + .unwrap(); + + let inside_build = Arc::new(Barrier::new(2)); + let release_build = Arc::new(Barrier::new(2)); + + let holder = { + let cache = cache.clone(); + let inside_build = inside_build.clone(); + let release_build = release_build.clone(); + thread::spawn(move || { + let hash = BlockHash::hash(&[9u8]); + let leaf = Sha256dHash::hash(&[0u8]); + cache + .get_or_build(holder_height, usize::MAX, + 100, + || Ok((hash, leaf)), + || { + inside_build.wait(); + release_build.wait(); + Ok((hash, build_levels_above_leaves(&[leaf])?, leaf)) + }, + ) + .unwrap(); + }) + }; + + inside_build.wait(); + + let result = cache.get_or_build(extra_height, usize::MAX, + 100, + || Ok((BlockHash::hash(&[11u8]), Sha256dHash::hash(&[0u8]))), + || panic!("must not rebuild once resident cache bytes and the held build already exhaust the budget"), + ); + assert!( + result.is_err(), + "already-resident cache bytes must count against the budget available to concurrent rebuilds" + ); + + release_build.wait(); + holder.join().unwrap(); } } diff --git a/tests/common.rs b/tests/common.rs index f6010fd8e..a18cb778a 100644 --- a/tests/common.rs +++ b/tests/common.rs @@ -131,6 +131,7 @@ impl TestRunner { electrum_txs_limit: 100, electrum_subscription_limit: 10_000, electrum_checkpoint_proof_concurrency_limit: 2, + electrum_checkpoint_merkle_cache_mb: 256, electrum_banner: "".into(), rpc_logging: RpcLogging::default(), zmq_addr: None, diff --git a/tests/config.rs b/tests/config.rs index 82e8f1eec..862cee1c2 100644 --- a/tests/config.rs +++ b/tests/config.rs @@ -80,3 +80,35 @@ fn startup_debug_log_identifies_cookie_file() { stderr ); } + +#[test] +fn checkpoint_proof_concurrency_limit_defaults_to_half_the_cpu_cores() { + let temp_dir = tempfile::tempdir().unwrap(); + let output = run_electrs(temp_dir.path(), &["-vv"]); + let stderr = String::from_utf8(output.stderr).unwrap(); + + let expected = (num_cpus::get() / 2).max(1); + let needle = format!("electrum_checkpoint_proof_concurrency_limit: {}", expected); + assert!( + stderr.contains(&needle), + "expected '{}' in stderr, got: {}", + needle, + stderr + ); +} + +#[test] +fn checkpoint_proof_concurrency_limit_can_be_overridden() { + let temp_dir = tempfile::tempdir().unwrap(); + let output = run_electrs( + temp_dir.path(), + &["-vv", "--electrum-checkpoint-proof-concurrency-limit", "7"], + ); + let stderr = String::from_utf8(output.stderr).unwrap(); + + assert!( + stderr.contains("electrum_checkpoint_proof_concurrency_limit: 7"), + "explicit override was not honored, stderr: {}", + stderr + ); +} From 6d4ed16ea99337cb8761ba295038fbc71354e6af Mon Sep 17 00:00:00 2001 From: Oleksandr Fedorenko Date: Mon, 28 Sep 2026 14:04:49 +0300 Subject: [PATCH 20/28] fix checkpoint Merkle cache eviction and reorg races --- src/electrum/server.rs | 63 ++-- src/util/electrum_merkle.rs | 625 +++++++++++++++++++++++++++++------- tests/electrum.rs | 249 ++++++++++++++ 3 files changed, 809 insertions(+), 128 deletions(-) diff --git a/src/electrum/server.rs b/src/electrum/server.rs index 39745567b..f79749504 100644 --- a/src/electrum/server.rs +++ b/src/electrum/server.rs @@ -277,22 +277,33 @@ impl Connection { let height = usize_from_value(params.get(0), "height")?; let cp_height = usize_from_value_or(params.get(1), "cp_height", 0)?; - let raw_header_hex: String = self - .query - .chain() - .header_by_height(height) - .map(|entry| serialize_hex(entry.header())) - .chain_err(|| "missing header")?; - if cp_height == 0 { + let raw_header_hex: String = self + .query + .chain() + .header_by_height(height) + .map(|entry| serialize_hex(entry.header())) + .chain_err(|| "missing header")?; return Ok(json!(raw_header_hex)); } - let (branch, root) = get_header_merkle_proof( + let (branch, root, proof_cp_hash) = get_header_merkle_proof( self.query.chain(), height, cp_height, self.checkpoint_proof_concurrency_limit, )?; + let indexed_headers = self.query.chain().store().headers(); + ensure!( + indexed_headers + .header_by_height(cp_height) + .map(|entry| entry.hash()) + == Some(&proof_cp_hash), + "chain changed while creating checkpoint merkle proof" + ); + let raw_header_hex = indexed_headers + .header_by_height(height) + .map(|entry| serialize_hex(entry.header())) + .chain_err(|| "missing header")?; Ok(json!({ "header": raw_header_hex, @@ -305,18 +316,15 @@ impl Connection { let start_height = usize_from_value(params.get(0), "start_height")?; let count = MAX_HEADERS.min(usize_from_value(params.get(1), "count")?); let cp_height = usize_from_value_or(params.get(2), "cp_height", 0)?; - let heights: Vec = (start_height..(start_height + count)).collect(); - let headers: Vec = heights - .into_iter() - .filter_map(|height| { - self.query - .chain() - .header_by_height(height) - .map(|entry| serialize_hex(entry.header())) - }) - .collect(); - if count == 0 || cp_height == 0 { + let headers: Vec = (start_height..(start_height + count)) + .filter_map(|height| { + self.query + .chain() + .header_by_height(height) + .map(|entry| serialize_hex(entry.header())) + }) + .collect(); return Ok(json!({ "count": headers.len(), "hex": headers.join(""), @@ -324,12 +332,27 @@ impl Connection { })); } - let (branch, root) = get_header_merkle_proof( + let (branch, root, proof_cp_hash) = get_header_merkle_proof( self.query.chain(), start_height + (count - 1), cp_height, self.checkpoint_proof_concurrency_limit, )?; + let indexed_headers = self.query.chain().store().headers(); + ensure!( + indexed_headers + .header_by_height(cp_height) + .map(|entry| entry.hash()) + == Some(&proof_cp_hash), + "chain changed while creating checkpoint merkle proof" + ); + let headers: Vec = (start_height..(start_height + count)) + .filter_map(|height| { + indexed_headers + .header_by_height(height) + .map(|entry| serialize_hex(entry.header())) + }) + .collect(); Ok(json!({ "count": headers.len(), diff --git a/src/util/electrum_merkle.rs b/src/util/electrum_merkle.rs index 716a61e56..534958585 100644 --- a/src/util/electrum_merkle.rs +++ b/src/util/electrum_merkle.rs @@ -100,13 +100,58 @@ fn estimated_peak_build_bytes(cp_height: usize) -> usize { .saturating_mul(3) } -type BuildSlot = Mutex>>; +struct MemoryCharge { + counter: Arc, + bytes: usize, +} + +impl MemoryCharge { + fn new(counter: Arc, bytes: usize) -> Self { + counter.fetch_add(bytes, Ordering::AcqRel); + Self { counter, bytes } + } +} + +impl Drop for MemoryCharge { + fn drop(&mut self) { + self.counter.fetch_sub(self.bytes, Ordering::AcqRel); + } +} + +struct RetainedTree { + tree: Arc, + _charge: MemoryCharge, +} + +struct BuildSlot { + waiters: AtomicUsize, + last_built: Mutex>, +} + +struct BuildSlotGuard<'a> { + cache: &'a CheckpointMerkleCache, + cp_height: usize, + slot: Arc, +} + +impl<'a> Drop for BuildSlotGuard<'a> { + fn drop(&mut self) { + let mut locks = self.cache.build_locks.lock().unwrap(); + if self.slot.waiters.fetch_sub(1, Ordering::AcqRel) == 1 { + if let Some(existing) = locks.get(&self.cp_height) { + if Arc::ptr_eq(existing, &self.slot) { + locks.remove(&self.cp_height); + } + } + } + } +} pub struct CheckpointMerkleCache { entries: Mutex)>>, build_locks: Mutex>>, inflight_rebuilds: AtomicUsize, - inflight_build_bytes: AtomicUsize, + inflight_build_bytes: Arc, capacity_bytes: usize, } @@ -116,7 +161,7 @@ impl CheckpointMerkleCache { entries: Mutex::new(Vec::new()), build_locks: Mutex::new(HashMap::new()), inflight_rebuilds: AtomicUsize::new(0), - inflight_build_bytes: AtomicUsize::new(0), + inflight_build_bytes: Arc::new(AtomicUsize::new(0)), capacity_bytes, } } @@ -135,44 +180,63 @@ impl CheckpointMerkleCache { let mut entries = self.entries.lock().unwrap(); entries.retain(|(h, _)| *h != cp_height && *h <= best_height); + if cp_height > best_height { + return; + } let mut total_bytes: usize = entries.iter().map(|(_, c)| c.size_bytes).sum(); + let mut remaining_entries = entries.len(); + let mut eviction_positions: Vec = entries + .iter() + .enumerate() + .filter_map(|(pos, (height, _))| (*height < cp_height).then_some(pos)) + .collect(); + eviction_positions.sort_unstable_by_key(|pos| entries[*pos].0); + + let mut evictions_needed = 0; while total_bytes + cached.size_bytes > self.capacity_bytes - || entries.len() >= MAX_CACHE_ENTRIES + || remaining_entries >= MAX_CACHE_ENTRIES { - match entries.iter().enumerate().min_by_key(|(_, (h, _))| *h) { - Some((pos, (min_height, _))) if cp_height > *min_height => { - total_bytes -= entries[pos].1.size_bytes; - entries.remove(pos); - } - _ => return, - } + let Some(pos) = eviction_positions.get(evictions_needed).copied() else { + return; + }; + total_bytes -= entries[pos].1.size_bytes; + remaining_entries -= 1; + evictions_needed += 1; + } + + let mut selected_positions = eviction_positions[..evictions_needed].to_vec(); + selected_positions.sort_unstable_by(|a, b| b.cmp(a)); + for pos in selected_positions { + entries.remove(pos); } entries.push((cp_height, cached)); } - fn build_lock_for(&self, cp_height: usize) -> Arc { + fn build_lock_for(&self, cp_height: usize) -> BuildSlotGuard<'_> { let mut locks = self.build_locks.lock().unwrap(); - locks + let slot = locks .entry(cp_height) - .or_insert_with(|| Arc::new(Mutex::new(None))) - .clone() - } - - fn release_build_lock(&self, cp_height: usize, lock: &Arc) { - let mut locks = self.build_locks.lock().unwrap(); - if let Some(existing) = locks.get(&cp_height) { - if Arc::ptr_eq(existing, lock) && Arc::strong_count(existing) <= 2 { - locks.remove(&cp_height); - } + .or_insert_with(|| { + Arc::new(BuildSlot { + waiters: AtomicUsize::new(0), + last_built: Mutex::new(None), + }) + }) + .clone(); + slot.waiters.fetch_add(1, Ordering::AcqRel); + BuildSlotGuard { + cache: self, + cp_height, + slot, } } fn get_or_build( &self, cp_height: usize, - best_height: usize, + current_best_height: impl FnOnce() -> usize, concurrency_limit: usize, snapshot_cp_hash_and_sibling: impl Fn() -> Result<(BlockHash, Sha256dHash)>, build: impl FnOnce() -> Result<(BlockHash, Vec>, Sha256dHash)>, @@ -183,36 +247,37 @@ impl CheckpointMerkleCache { } let build_lock = self.build_lock_for(cp_height); - let result = self.build_under_lock( + self.build_under_lock( cp_height, - best_height, + current_best_height, cp_hash, sibling, concurrency_limit, - &build_lock, + &build_lock.slot, build, - ); - self.release_build_lock(cp_height, &build_lock); - result + ) } fn build_under_lock( &self, cp_height: usize, - best_height: usize, + current_best_height: impl FnOnce() -> usize, cp_hash: BlockHash, sibling: Sha256dHash, concurrency_limit: usize, build_lock: &BuildSlot, build: impl FnOnce() -> Result<(BlockHash, Vec>, Sha256dHash)>, ) -> Result<(Arc, Sha256dHash)> { - let mut last_built = build_lock.lock().unwrap(); + let mut last_built = build_lock + .last_built + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); if let Some(cached) = self.get_valid(cp_height, cp_hash) { return Ok((cached, sibling)); } - if let Some(cached) = last_built.as_ref().filter(|c| c.cp_hash == cp_hash) { - return Ok((cached.clone(), sibling)); + if let Some(retained) = last_built.as_ref().filter(|r| r.tree.cp_hash == cp_hash) { + return Ok((retained.tree.clone(), sibling)); } let _permit = RebuildPermit::acquire(&self.inflight_rebuilds, concurrency_limit)?; @@ -226,14 +291,19 @@ impl CheckpointMerkleCache { drop(entries); let (cp_hash, levels, sibling) = build()?; let cached = Arc::new(CachedLevels::new(cp_hash, levels)); - self.insert(cp_height, best_height, cached.clone()); - *last_built = Some(cached.clone()); + self.insert(cp_height, current_best_height(), cached.clone()); + *last_built = Some(RetainedTree { + _charge: MemoryCharge::new(self.inflight_build_bytes.clone(), cached.size_bytes), + tree: cached.clone(), + }); Ok((cached, sibling)) } } fn merklize(left: Sha256dHash, right: Sha256dHash) -> Sha256dHash { - let data = [&left[..], &right[..]].concat(); + let mut data = [0u8; 64]; + data[..32].copy_from_slice(&left[..]); + data[32..].copy_from_slice(&right[..]); Sha256dHash::hash(&data) } @@ -310,7 +380,7 @@ pub fn get_header_merkle_proof( height: usize, cp_height: usize, checkpoint_proof_concurrency_limit: usize, -) -> Result<(Vec, Sha256dHash)> { +) -> Result<(Vec, Sha256dHash, BlockHash)> { if cp_height < height { bail!("cp_height #{} < height #{}", cp_height, height); } @@ -326,16 +396,19 @@ pub fn get_header_merkle_proof( if cp_height == 0 { let headers = chain.store().headers(); - let root = Sha256dHash::from( - *headers - .header_by_height(0) - .chain_err(|| "missing block header at height 0")? - .hash(), - ); - return Ok((vec![], root)); + let cp_hash = *headers + .header_by_height(0) + .chain_err(|| "missing block header at height 0")? + .hash(); + let root = Sha256dHash::from(cp_hash); + return Ok((vec![], root, cp_hash)); } - let sibling_index = if height % 2 == 0 { height + 1 } else { height - 1 }; + let sibling_index = if height % 2 == 0 { + height + 1 + } else { + height - 1 + }; let sibling_height = sibling_index.min(cp_height); let snapshot_cp_hash_and_sibling = || -> Result<(BlockHash, Sha256dHash)> { @@ -374,14 +447,16 @@ pub fn get_header_merkle_proof( Ok((cp_hash, levels, sibling)) }; - let (cached, sibling) = chain.checkpoint_merkle_cache().get_or_build(cp_height, - best_height, + let (cached, sibling) = chain.checkpoint_merkle_cache().get_or_build( + cp_height, + || chain.best_height(), checkpoint_proof_concurrency_limit, snapshot_cp_hash_and_sibling, build, )?; - extract_branch_and_root(&cached.levels, sibling, height) + let (branch, root) = extract_branch_and_root(&cached.levels, sibling, height)?; + Ok((branch, root, cached.cp_hash)) } #[trace] @@ -447,7 +522,11 @@ mod tests { } fn sibling_leaf(leaves: &[Sha256dHash], cp_height: usize, height: usize) -> Sha256dHash { - let sibling_index = if height % 2 == 0 { height + 1 } else { height - 1 }; + let sibling_index = if height % 2 == 0 { + height + 1 + } else { + height - 1 + }; leaves[sibling_index.min(cp_height)] } @@ -495,7 +574,9 @@ mod tests { for height in [0usize, 3, 8] { let sibling = sibling_leaf(&leaves, cp_height, height); let (cached, _) = cache - .get_or_build(cp_height, usize::MAX, + .get_or_build( + cp_height, + || usize::MAX, 1, || Ok((cp_hash, sibling)), || { @@ -530,24 +611,42 @@ mod tests { let build_count = AtomicUsize::new(0); cache - .get_or_build(cp_height, usize::MAX, 4, || Ok((old_hash, sibling)), || { - build_count.fetch_add(1, Ordering::SeqCst); - Ok((old_hash, build_levels_above_leaves(&leaves)?, sibling)) - }) + .get_or_build( + cp_height, + || usize::MAX, + 4, + || Ok((old_hash, sibling)), + || { + build_count.fetch_add(1, Ordering::SeqCst); + Ok((old_hash, build_levels_above_leaves(&leaves)?, sibling)) + }, + ) .unwrap(); cache - .get_or_build(cp_height, usize::MAX, 4, || Ok((old_hash, sibling)), || { - build_count.fetch_add(1, Ordering::SeqCst); - Ok((old_hash, build_levels_above_leaves(&leaves)?, sibling)) - }) + .get_or_build( + cp_height, + || usize::MAX, + 4, + || Ok((old_hash, sibling)), + || { + build_count.fetch_add(1, Ordering::SeqCst); + Ok((old_hash, build_levels_above_leaves(&leaves)?, sibling)) + }, + ) .unwrap(); assert_eq!(build_count.load(Ordering::SeqCst), 1); cache - .get_or_build(cp_height, usize::MAX, 4, || Ok((new_hash, sibling)), || { - build_count.fetch_add(1, Ordering::SeqCst); - Ok((new_hash, build_levels_above_leaves(&leaves)?, sibling)) - }) + .get_or_build( + cp_height, + || usize::MAX, + 4, + || Ok((new_hash, sibling)), + || { + build_count.fetch_add(1, Ordering::SeqCst); + Ok((new_hash, build_levels_above_leaves(&leaves)?, sibling)) + }, + ) .unwrap(); assert_eq!( build_count.load(Ordering::SeqCst), @@ -559,7 +658,8 @@ mod tests { #[test] fn stale_entry_returns_correct_branch_and_root_after_checkpoint_hash_changes() { let old_leaves = leaves(5); - let new_leaves: Vec = (100..105).map(|i| Sha256dHash::hash(&[i as u8])).collect(); + let new_leaves: Vec = + (100..105).map(|i| Sha256dHash::hash(&[i as u8])).collect(); let cp_height = 4; let height = 1; let old_sibling = sibling_leaf(&old_leaves, cp_height, height); @@ -570,9 +670,19 @@ mod tests { let cache = CheckpointMerkleCache::new(GENEROUS_TEST_CACHE_BYTES); let (old_cached, old_ret_sibling) = cache - .get_or_build(cp_height, usize::MAX, 4, || Ok((old_hash, old_sibling)), || { - Ok((old_hash, build_levels_above_leaves(&old_leaves)?, old_sibling)) - }) + .get_or_build( + cp_height, + || usize::MAX, + 4, + || Ok((old_hash, old_sibling)), + || { + Ok(( + old_hash, + build_levels_above_leaves(&old_leaves)?, + old_sibling, + )) + }, + ) .unwrap(); let (old_branch, old_root) = extract_branch_and_root(&old_cached.levels, old_ret_sibling, height).unwrap(); @@ -582,9 +692,19 @@ mod tests { assert_eq!(old_root, expected_old_root); let (new_cached, new_ret_sibling) = cache - .get_or_build(cp_height, usize::MAX, 4, || Ok((new_hash, new_sibling)), || { - Ok((new_hash, build_levels_above_leaves(&new_leaves)?, new_sibling)) - }) + .get_or_build( + cp_height, + || usize::MAX, + 4, + || Ok((new_hash, new_sibling)), + || { + Ok(( + new_hash, + build_levels_above_leaves(&new_leaves)?, + new_sibling, + )) + }, + ) .unwrap(); let (new_branch, new_root) = extract_branch_and_root(&new_cached.levels, new_ret_sibling, height).unwrap(); @@ -609,7 +729,9 @@ mod tests { for cp_height in [10usize, 20, 30, 40, 50] { let hash = BlockHash::hash(&[cp_height as u8]); cache - .get_or_build(cp_height, usize::MAX, + .get_or_build( + cp_height, + || usize::MAX, 4, || Ok((hash, leaf)), || Ok((hash, build_levels_above_leaves(&entry_leaves)?, leaf)), @@ -620,7 +742,9 @@ mod tests { let hash60 = BlockHash::hash(&[60u8]); cache - .get_or_build(60, usize::MAX, + .get_or_build( + 60, + || usize::MAX, 4, || Ok((hash60, leaf)), || Ok((hash60, build_levels_above_leaves(&entry_leaves)?, leaf)), @@ -654,7 +778,9 @@ mod tests { for cp_height in [10usize, 20, 30, 40, 50] { let hash = BlockHash::hash(&[cp_height as u8]); cache - .get_or_build(cp_height, usize::MAX, + .get_or_build( + cp_height, + || usize::MAX, 4, || Ok((hash, leaf)), || Ok((hash, build_levels_above_leaves(&entry_leaves)?, leaf)), @@ -665,7 +791,9 @@ mod tests { let hash5 = BlockHash::hash(&[5u8]); cache - .get_or_build(5, usize::MAX, + .get_or_build( + 5, + || usize::MAX, 4, || Ok((hash5, leaf)), || Ok((hash5, build_levels_above_leaves(&entry_leaves)?, leaf)), @@ -688,6 +816,31 @@ mod tests { assert!(!heights.contains(&5)); } + #[test] + fn failed_insert_does_not_partially_evict_lower_entries() { + let hash_bytes = std::mem::size_of::(); + let cache = CheckpointMerkleCache::new(5 * hash_bytes); + let make_cached = |height: usize, hashes: usize| { + Arc::new(CachedLevels::new( + BlockHash::hash(&[height as u8]), + vec![vec![Sha256dHash::hash(&[height as u8]); hashes]], + )) + }; + + cache.insert(10, usize::MAX, make_cached(10, 1)); + cache.insert(30, usize::MAX, make_cached(30, 4)); + cache.insert(20, usize::MAX, make_cached(20, 2)); + + let heights: Vec = cache + .entries + .lock() + .unwrap() + .iter() + .map(|(height, _)| *height) + .collect(); + assert_eq!(heights, vec![10, 30]); + } + #[test] fn insert_caps_entry_count_even_under_a_generous_byte_budget() { let entry_leaves = leaves(2); @@ -699,7 +852,9 @@ mod tests { for cp_height in 0..MAX_CACHE_ENTRIES + 50 { cache - .get_or_build(cp_height, usize::MAX, + .get_or_build( + cp_height, + || usize::MAX, 4, || Ok((hash, leaf)), || Ok((hash, build_levels_above_leaves(&entry_leaves)?, leaf)), @@ -726,9 +881,13 @@ mod tests { let old_height = 100; let old_hash = BlockHash::hash(&[1u8]); cache - .get_or_build(old_height, old_height, 4, || Ok((old_hash, leaf)), || { - Ok((old_hash, build_levels_above_leaves(&entry_leaves)?, leaf)) - }) + .get_or_build( + old_height, + || old_height, + 4, + || Ok((old_hash, leaf)), + || Ok((old_hash, build_levels_above_leaves(&entry_leaves)?, leaf)), + ) .unwrap(); assert_eq!(cache.entries.lock().unwrap().len(), 1); @@ -737,18 +896,30 @@ mod tests { let build_count = AtomicUsize::new(0); cache - .get_or_build(new_height, new_height, 4, || Ok((new_hash, leaf)), || { - build_count.fetch_add(1, Ordering::SeqCst); - Ok((new_hash, build_levels_above_leaves(&entry_leaves)?, leaf)) - }) + .get_or_build( + new_height, + || new_height, + 4, + || Ok((new_hash, leaf)), + || { + build_count.fetch_add(1, Ordering::SeqCst); + Ok((new_hash, build_levels_above_leaves(&entry_leaves)?, leaf)) + }, + ) .unwrap(); assert_eq!(build_count.load(Ordering::SeqCst), 1); cache - .get_or_build(new_height, new_height, 4, || Ok((new_hash, leaf)), || { - build_count.fetch_add(1, Ordering::SeqCst); - panic!("must serve from cache instead of rebuilding") - }) + .get_or_build( + new_height, + || new_height, + 4, + || Ok((new_hash, leaf)), + || { + build_count.fetch_add(1, Ordering::SeqCst); + panic!("must serve from cache instead of rebuilding") + }, + ) .unwrap(); assert_eq!( build_count.load(Ordering::SeqCst), @@ -767,6 +938,54 @@ mod tests { assert_eq!(heights, vec![new_height]); } + #[test] + fn insertion_reads_best_height_after_a_slow_rebuild() { + let entry_leaves = leaves(2); + let levels = build_levels_above_leaves(&entry_leaves).unwrap(); + let entry_bytes = levels[0].len() * std::mem::size_of::(); + let cache = CheckpointMerkleCache::new(2 * entry_bytes); + let leaf = entry_leaves[0]; + + let new_tip_height = 101; + cache.insert( + new_tip_height, + new_tip_height, + Arc::new(CachedLevels::new( + BlockHash::hash(&[new_tip_height as u8]), + levels.clone(), + )), + ); + + let best_height = AtomicUsize::new(new_tip_height - 1); + let cp_height = new_tip_height - 1; + let cp_hash = BlockHash::hash(&[cp_height as u8]); + cache + .get_or_build( + cp_height, + || best_height.load(Ordering::SeqCst), + 1, + || Ok((cp_hash, leaf)), + || { + best_height.store(new_tip_height, Ordering::SeqCst); + Ok((cp_hash, levels, leaf)) + }, + ) + .unwrap(); + + let heights: Vec = cache + .entries + .lock() + .unwrap() + .iter() + .map(|(height, _)| *height) + .collect(); + assert!( + heights.contains(&new_tip_height), + "a tip cached during the rebuild must not be pruned using the rebuild's starting height" + ); + assert!(heights.contains(&cp_height)); + } + #[test] fn concurrent_rebuilds_of_same_cp_height_run_build_once() { let leaves = Arc::new(leaves(9)); @@ -787,7 +1006,9 @@ mod tests { thread::spawn(move || { barrier.wait(); cache - .get_or_build(cp_height, usize::MAX, + .get_or_build( + cp_height, + || usize::MAX, 8, || Ok((cp_hash, sibling)), || { @@ -810,6 +1031,10 @@ mod tests { 1, "8 concurrent requests for the same cp_height must build exactly once" ); + assert!( + cache.build_locks.lock().unwrap().is_empty(), + "no build slot may remain after all requests finished" + ); } #[test] @@ -833,7 +1058,9 @@ mod tests { thread::spawn(move || { barrier.wait(); cache - .get_or_build(cp_height, usize::MAX, + .get_or_build( + cp_height, + || usize::MAX, 8, || Ok((cp_hash, sibling)), || { @@ -856,6 +1083,158 @@ mod tests { 1, "8 concurrent requests for the same uncacheable cp_height must still build exactly once" ); + drop(_keep_build_slot_alive); + assert!( + cache.build_locks.lock().unwrap().is_empty(), + "no build slot may remain once the last request releases it" + ); + } + + #[test] + fn concurrent_releases_must_not_leak_the_build_slot_and_its_tree() { + let leaves = leaves(9); + let cp_height = 8; + let cp_hash = BlockHash::hash(&[9u8]); + let sibling = sibling_leaf(&leaves, cp_height, 0); + + let cache = CheckpointMerkleCache::new(0); + + let slot_a = cache.build_lock_for(cp_height); + let slot_b = cache.build_lock_for(cp_height); + + cache + .build_under_lock( + cp_height, + || usize::MAX, + cp_hash, + sibling, + 1, + &slot_a.slot, + || Ok((cp_hash, build_levels_above_leaves(&leaves)?, sibling)), + ) + .unwrap(); + + drop(slot_a); + assert_eq!( + cache.build_locks.lock().unwrap().len(), + 1, + "slot must stay while another request still holds it" + ); + drop(slot_b); + + let locks = cache.build_locks.lock().unwrap(); + assert!( + locks.is_empty(), + "build slot leaked after all requests finished, retaining the tree \ + outside the cache's memory accounting" + ); + } + + #[test] + fn tree_retained_only_by_a_build_slot_counts_against_the_memory_budget() { + let entry_leaves = leaves(2); + let retained_bytes = build_levels_above_leaves(&entry_leaves).unwrap()[0].len() + * std::mem::size_of::(); + let leaf = entry_leaves[0]; + + let blocked_height = 1_000usize; + let cache = CheckpointMerkleCache::new(estimated_peak_build_bytes(blocked_height)); + + let uncached_height = 100; + let uncached_hash = BlockHash::hash(&[1u8]); + let slot_guard = cache.build_lock_for(uncached_height); + cache + .build_under_lock( + uncached_height, + || uncached_height - 1, + uncached_hash, + leaf, + 4, + &slot_guard.slot, + || { + Ok(( + uncached_hash, + build_levels_above_leaves(&entry_leaves)?, + leaf, + )) + }, + ) + .unwrap(); + assert!(cache.entries.lock().unwrap().is_empty()); + assert_eq!( + cache.inflight_build_bytes.load(Ordering::SeqCst), + retained_bytes, + "a tree retained only by last_built must be charged to the budget" + ); + + let blocked_hash = BlockHash::hash(&[2u8]); + let result = cache.get_or_build( + blocked_height, + || usize::MAX, + 4, + || Ok((blocked_hash, leaf)), + || panic!("must not build while retained slot bytes exhaust the budget"), + ); + assert!( + result.is_err(), + "a rebuild must be rejected while slot-retained bytes exhaust the budget" + ); + + drop(slot_guard); + assert_eq!( + cache.inflight_build_bytes.load(Ordering::SeqCst), + 0, + "releasing the slot must release its memory charge" + ); + cache + .get_or_build( + blocked_height, + || usize::MAX, + 4, + || Ok((blocked_hash, leaf)), + || Ok((blocked_hash, build_levels_above_leaves(&entry_leaves)?, leaf)), + ) + .unwrap(); + } + + #[test] + fn a_panicking_build_does_not_poison_the_slot_for_waiters() { + let leaves = Arc::new(leaves(5)); + let cp_height = 4; + let cp_hash = BlockHash::hash(&[1u8]); + let sibling = sibling_leaf(&leaves, cp_height, 0); + + let cache = Arc::new(CheckpointMerkleCache::new(GENEROUS_TEST_CACHE_BYTES)); + let _keep_slot_alive = cache.build_lock_for(cp_height); + + let panicking = { + let cache = cache.clone(); + thread::spawn(move || { + let _ = cache.get_or_build( + cp_height, + || usize::MAX, + 4, + || Ok((cp_hash, sibling)), + || panic!("simulated build failure"), + ); + }) + }; + assert!(panicking.join().is_err()); + assert_eq!( + cache.build_locks.lock().unwrap().len(), + 1, + "the slot must survive for the still-registered request" + ); + + cache + .get_or_build( + cp_height, + || usize::MAX, + 4, + || Ok((cp_hash, sibling)), + || Ok((cp_hash, build_levels_above_leaves(&leaves)?, sibling)), + ) + .expect("a waiter must recover from a poisoned slot instead of panicking"); } #[test] @@ -886,15 +1265,23 @@ mod tests { let cache = CheckpointMerkleCache::new(GENEROUS_TEST_CACHE_BYTES); cache - .get_or_build(cp_height, usize::MAX, 1, || Ok((cp_hash, sibling)), || { - Ok((cp_hash, build_levels_above_leaves(&leaves)?, sibling)) - }) + .get_or_build( + cp_height, + || usize::MAX, + 1, + || Ok((cp_hash, sibling)), + || Ok((cp_hash, build_levels_above_leaves(&leaves)?, sibling)), + ) .unwrap(); cache - .get_or_build(cp_height, usize::MAX, 0, || Ok((cp_hash, sibling)), || { - panic!("must not rebuild on a cache hit") - }) + .get_or_build( + cp_height, + || usize::MAX, + 0, + || Ok((cp_hash, sibling)), + || panic!("must not rebuild on a cache hit"), + ) .unwrap(); } @@ -907,9 +1294,13 @@ mod tests { let cache = Arc::new(CheckpointMerkleCache::new(GENEROUS_TEST_CACHE_BYTES)); cache - .get_or_build(cp_height, usize::MAX, 1, || Ok((cp_hash, sibling)), || { - Ok((cp_hash, build_levels_above_leaves(&leaves)?, sibling)) - }) + .get_or_build( + cp_height, + || usize::MAX, + 1, + || Ok((cp_hash, sibling)), + || Ok((cp_hash, build_levels_above_leaves(&leaves)?, sibling)), + ) .unwrap(); const HOLDERS: usize = 2; @@ -926,7 +1317,9 @@ mod tests { let hash = BlockHash::hash(&[100 + i as u8]); let leaf = Sha256dHash::hash(&[0u8]); cache - .get_or_build(miss_height, usize::MAX, + .get_or_build( + miss_height, + || usize::MAX, HOLDERS, || Ok((hash, leaf)), || { @@ -943,9 +1336,13 @@ mod tests { inside_build.wait(); cache - .get_or_build(cp_height, usize::MAX, 0, || Ok((cp_hash, sibling)), || { - panic!("must not rebuild on a cache hit") - }) + .get_or_build( + cp_height, + || usize::MAX, + 0, + || Ok((cp_hash, sibling)), + || panic!("must not rebuild on a cache hit"), + ) .unwrap(); release_build.wait(); @@ -977,7 +1374,9 @@ mod tests { let hash = BlockHash::hash(&[miss_height as u8]); let leaf = Sha256dHash::hash(&[0u8]); cache - .get_or_build(miss_height, usize::MAX, + .get_or_build( + miss_height, + || usize::MAX, 100, || Ok((hash, leaf)), || { @@ -993,9 +1392,13 @@ mod tests { inside_build.wait(); - let result = cache.get_or_build(3_000, usize::MAX, 100, || Ok((BlockHash::hash(&[9u8]), Sha256dHash::hash(&[0u8]))), || { - panic!("must not rebuild once the memory budget is fully held") - }); + let result = cache.get_or_build( + 3_000, + || usize::MAX, + 100, + || Ok((BlockHash::hash(&[9u8]), Sha256dHash::hash(&[0u8]))), + || panic!("must not rebuild once the memory budget is fully held"), + ); assert!( result.is_err(), "a third concurrent rebuild must be rejected once the memory budget is exhausted, even with a generous concurrency_limit" @@ -1025,7 +1428,9 @@ mod tests { let cache = Arc::new(CheckpointMerkleCache::new(capacity_bytes)); cache - .get_or_build(resident_height, usize::MAX, + .get_or_build( + resident_height, + || usize::MAX, 1, || Ok((resident_hash, resident_sibling)), || { @@ -1049,7 +1454,9 @@ mod tests { let hash = BlockHash::hash(&[9u8]); let leaf = Sha256dHash::hash(&[0u8]); cache - .get_or_build(holder_height, usize::MAX, + .get_or_build( + holder_height, + || usize::MAX, 100, || Ok((hash, leaf)), || { @@ -1064,7 +1471,9 @@ mod tests { inside_build.wait(); - let result = cache.get_or_build(extra_height, usize::MAX, + let result = cache.get_or_build( + extra_height, + || usize::MAX, 100, || Ok((BlockHash::hash(&[11u8]), Sha256dHash::hash(&[0u8]))), || panic!("must not rebuild once resident cache bytes and the held build already exhaust the budget"), diff --git a/tests/electrum.rs b/tests/electrum.rs index 3f064f575..da8db0188 100644 --- a/tests/electrum.rs +++ b/tests/electrum.rs @@ -445,6 +445,255 @@ fn test_electrum_broadcast_package_updates_mempool() -> Result<()> { Ok(()) } +#[cfg(not(feature = "liquid"))] +mod checkpoint { + use bitcoin::hashes::{sha256d, Hash}; + + pub fn merklize(left: sha256d::Hash, right: sha256d::Hash) -> sha256d::Hash { + let mut data = [0u8; 64]; + data[..32].copy_from_slice(&left[..]); + data[32..].copy_from_slice(&right[..]); + sha256d::Hash::hash(&data) + } + + pub fn merkle_root(mut hashes: Vec) -> sha256d::Hash { + assert!(!hashes.is_empty()); + while hashes.len() > 1 { + if hashes.len() % 2 != 0 { + hashes.push(*hashes.last().unwrap()); + } + hashes = hashes + .chunks(2) + .map(|pair| merklize(pair[0], pair[1])) + .collect(); + } + hashes[0] + } + + pub fn root_from_branch( + leaf: sha256d::Hash, + branch: &[sha256d::Hash], + mut index: usize, + ) -> sha256d::Hash { + let mut current = leaf; + for sibling in branch { + current = if index % 2 == 0 { + merklize(current, *sibling) + } else { + merklize(*sibling, current) + }; + index /= 2; + } + current + } +} + +#[cfg(not(feature = "liquid"))] +#[test] +fn test_electrum_checkpoint_proofs() -> Result<()> { + use bitcoin::hashes::{sha256d, Hash}; + use bitcoin::hex::FromHex; + use std::str::FromStr; + + let (_electrum_server, electrum_addr, tester) = common::init_electrum_tester()?; + let tip = tester.get_block_count()? as usize; + + let leaves: Vec = (0..=tip) + .map(|h| Ok(tester.get_block_hash(h as u64)?.to_raw_hash())) + .collect::>()?; + let expected_root = checkpoint::merkle_root(leaves.clone()); + + let mut stream = TcpStream::connect(electrum_addr).unwrap(); + let parse = |s: &str| -> electrumd::jsonrpc::serde_json::Value { + electrumd::jsonrpc::serde_json::from_str(s).unwrap() + }; + + let height = 5usize; + let v = parse(&write_and_read( + &mut stream, + &format!( + "{{\"jsonrpc\": \"2.0\", \"method\": \"blockchain.block.header\", \"params\": [{}, {}], \"id\": 1}}", + height, tip + ), + )); + let result = &v["result"]; + let header_bytes = + Vec::::from_hex(result["header"].as_str().expect("header hex")).unwrap(); + assert_eq!( + sha256d::Hash::hash(&header_bytes), + leaves[height], + "returned header must hash to the block hash at the requested height" + ); + let root = sha256d::Hash::from_str(result["root"].as_str().expect("root")).unwrap(); + assert_eq!(root, expected_root, "proof root must match the independent tree"); + let branch: Vec = result["branch"] + .as_array() + .expect("branch array") + .iter() + .map(|h| sha256d::Hash::from_str(h.as_str().unwrap()).unwrap()) + .collect(); + assert_eq!( + checkpoint::root_from_branch(leaves[height], &branch, height), + expected_root, + "branch must connect the leaf to the root" + ); + + let v2 = parse(&write_and_read( + &mut stream, + &format!( + "{{\"jsonrpc\": \"2.0\", \"method\": \"blockchain.block.header\", \"params\": [{}, {}], \"id\": 2}}", + height, tip + ), + )); + assert_eq!(v["result"], v2["result"], "cached proof must be identical"); + + let v = parse(&write_and_read( + &mut stream, + &format!( + "{{\"jsonrpc\": \"2.0\", \"method\": \"blockchain.block.header\", \"params\": [{}, 0], \"id\": 3}}", + height + ), + )); + assert!(v["result"].is_string(), "cp_height=0 must return bare hex"); + + let v = parse(&write_and_read( + &mut stream, + &format!( + "{{\"jsonrpc\": \"2.0\", \"method\": \"blockchain.block.header\", \"params\": [{}, {}], \"id\": 4}}", + height, height - 1 + ), + )); + assert!(!v["error"].is_null(), "cp_height < height must be an error"); + + let v = parse(&write_and_read( + &mut stream, + &format!( + "{{\"jsonrpc\": \"2.0\", \"method\": \"blockchain.block.header\", \"params\": [{}, {}], \"id\": 5}}", + height, tip + 1 + ), + )); + assert!(!v["error"].is_null(), "cp_height above the tip must be an error"); + + let (start, count) = (0usize, 10usize); + let v = parse(&write_and_read( + &mut stream, + &format!( + "{{\"jsonrpc\": \"2.0\", \"method\": \"blockchain.block.headers\", \"params\": [{}, {}, {}], \"id\": 6}}", + start, count, tip + ), + )); + let result = &v["result"]; + assert_eq!(result["count"].as_u64(), Some(count as u64)); + let hex = result["hex"].as_str().expect("headers hex"); + assert_eq!(hex.len(), count * 80 * 2); + let root = sha256d::Hash::from_str(result["root"].as_str().expect("root")).unwrap(); + assert_eq!(root, expected_root); + let branch: Vec = result["branch"] + .as_array() + .expect("branch array") + .iter() + .map(|h| sha256d::Hash::from_str(h.as_str().unwrap()).unwrap()) + .collect(); + let proof_height = start + count - 1; + assert_eq!( + checkpoint::root_from_branch(leaves[proof_height], &branch, proof_height), + expected_root, + "headers branch must connect the last requested header to the root" + ); + + let v = parse(&write_and_read( + &mut stream, + &format!( + "{{\"jsonrpc\": \"2.0\", \"method\": \"blockchain.block.headers\", \"params\": [{}, {}, {}], \"id\": 7}}", + tip - 1, 5, tip + ), + )); + assert!( + !v["error"].is_null(), + "a headers range extending past cp_height must be an error" + ); + + Ok(()) +} + +#[cfg(not(feature = "liquid"))] +#[test] +fn test_electrum_checkpoint_proof_reorg_consistency() -> Result<()> { + use bitcoin::hashes::{sha256d, Hash}; + use bitcoin::hex::FromHex; + use std::str::FromStr; + + let (_electrum_server, electrum_addr, mut tester) = common::init_electrum_tester()?; + let tip = tester.get_block_count()? as usize; + let height = 3usize; + + let mut stream = TcpStream::connect(electrum_addr).unwrap(); + let parse = |s: &str| -> electrumd::jsonrpc::serde_json::Value { + electrumd::jsonrpc::serde_json::from_str(s).unwrap() + }; + let request = |stream: &mut TcpStream, id: u32| { + write_and_read( + stream, + &format!( + "{{\"jsonrpc\": \"2.0\", \"method\": \"blockchain.block.header\", \"params\": [{}, {}], \"id\": {}}}", + height, tip, id + ), + ) + }; + + let v = parse(&request(&mut stream, 1)); + let old_root = sha256d::Hash::from_str(v["result"]["root"].as_str().expect("root")).unwrap(); + + let old_tip_hash = tester.get_block_hash(tip as u64)?; + let _: electrumd::jsonrpc::serde_json::Value = tester + .node_client() + .call("invalidateblock", &[json!(old_tip_hash.to_string())])?; + tester.mine()?; + tester.mine()?; + assert_ne!( + tester.get_block_hash(tip as u64)?, + old_tip_hash, + "the reorg must have replaced the block at the old tip height" + ); + + let v = parse(&request(&mut stream, 2)); + let result = &v["result"]; + let new_root = sha256d::Hash::from_str(result["root"].as_str().expect("root")).unwrap(); + assert_ne!( + new_root, old_root, + "a reorg past the checkpoint must invalidate the cached tree" + ); + + let leaves: Vec = (0..=tip) + .map(|h| Ok(tester.get_block_hash(h as u64)?.to_raw_hash())) + .collect::>()?; + assert_eq!( + new_root, + checkpoint::merkle_root(leaves.clone()), + "post-reorg root must match the new chain" + ); + let branch: Vec = result["branch"] + .as_array() + .expect("branch array") + .iter() + .map(|h| sha256d::Hash::from_str(h.as_str().unwrap()).unwrap()) + .collect(); + assert_eq!( + checkpoint::root_from_branch(leaves[height], &branch, height), + new_root, + "post-reorg branch must connect the leaf to the new root" + ); + let header_bytes = + Vec::::from_hex(result["header"].as_str().expect("header hex")).unwrap(); + assert_eq!( + sha256d::Hash::hash(&header_bytes), + leaves[height], + "post-reorg header must belong to the same snapshot as the proof" + ); + + Ok(()) +} + fn write_and_read(stream: &mut TcpStream, write: &str) -> String { stream.write_all(write.as_bytes()).unwrap(); stream.write(b"\n").unwrap(); From 14a0ed788702b47cffcc1c4cfea514c48864b30d Mon Sep 17 00:00:00 2001 From: Edward Houston Date: Thu, 24 Sep 2026 11:55:11 +0200 Subject: [PATCH 21/28] Avoid holding the mempool write lock during broadcast RPCs Fetch missing transactions through bounded proxied RPCs outside the mempool lock. Recheck presence before insertion, log unavailable txids, and cover lock behavior with a regression test. --- src/daemon.rs | 4 ++- src/metrics.rs | 5 +++ src/new_index/mempool.rs | 74 +++++++++++++++++++++++++++------------- src/new_index/query.rs | 12 ++----- tests/common.rs | 17 +++++++++ tests/rest.rs | 65 +++++++++++++++++++++++++++++++++-- 6 files changed, 141 insertions(+), 36 deletions(-) diff --git a/src/daemon.rs b/src/daemon.rs index 7e4442e73..d60b133aa 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -1385,9 +1385,11 @@ impl Daemon { self.request("getrawtransaction", json!([txid, verbose, blockhash])) } + /// Fetch a mempool transaction on behalf of an API client broadcast/package path. #[trace] pub fn getmempooltx(&self, txhash: &Txid) -> Result { - let value = self.request("getrawtransaction", json!([txhash, /*verbose=*/ false]))?; + let value = + self.request_proxied("getrawtransaction", json!([txhash, /*verbose=*/ false]))?; tx_from_value(value) } diff --git a/src/metrics.rs b/src/metrics.rs index e0c56ac75..e32dab068 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -72,6 +72,11 @@ impl Metrics { h } + /// Snapshot of every registered metric, e.g. for asserting on counters in tests. + pub fn gather(&self) -> Vec { + self.reg.gather() + } + pub fn start(&self) { let server = tiny_http::Server::http(self.addr) .unwrap_or_else(|_| panic!("failed to start monitoring HTTP server at {}", self.addr)); diff --git a/src/new_index/mempool.rs b/src/new_index/mempool.rs index 88f73cbc1..cf91763d8 100644 --- a/src/new_index/mempool.rs +++ b/src/new_index/mempool.rs @@ -315,41 +315,63 @@ impl Mempool { self.backlog_stats = (BacklogStats::from_feeinfo_slice(&feeinfo), Instant::now()); } + /// Ensure a single transaction (e.g. a broadcast one) is in the mempool. Fails if the txid is not + /// already indexed and cannot be fetched from the daemon. #[trace] - pub fn add_by_txid(&mut self, daemon: &Daemon, txid: Txid) -> Result<()> { - if self.txstore.get(&txid).is_none() { - if let Ok(tx) = daemon.getmempooltx(&txid) { - let mut txs_map = HashMap::new(); - txs_map.insert(txid, tx); - self.add(txs_map) - } else { - bail!("add_by_txid cannot find {}", txid); - } - } else { - Ok(()) + pub fn ensure_tx(mempool: &RwLock, daemon: &Daemon, txid: Txid) -> Result<()> { + if !Self::add_missing(mempool, daemon, &[txid])?.is_empty() { + bail!("ensure_tx cannot find txid='{}'", txid); } + Ok(()) } - /// Add multiple transactions (e.g. an accepted package) to the mempool in a single batch, + /// Ensure multiple transactions (e.g. an accepted package) are in the mempool in a single batch, /// so that interdependent parent/child txs are linked within the same `add` call. Txids that /// are already present or cannot be fetched from the daemon are skipped. - pub fn add_by_txids(&mut self, daemon: &Daemon, txids: &[Txid]) -> Result<()> { + pub fn ensure_txs(mempool: &RwLock, daemon: &Daemon, txids: &[Txid]) -> Result<()> { + Self::add_missing(mempool, daemon, txids).map(|_| ()) + } + + /// Fetch and index the given txids that are not yet in the local mempool, returning the ones + /// that could not be fetched. The daemon RPCs run without holding the mempool lock, so they + /// never block concurrent readers; the write lock is only taken to insert. + fn add_missing( + mempool: &RwLock, + daemon: &Daemon, + txids: &[Txid], + ) -> Result> { + let missing: Vec = { + let mempool = mempool.read().unwrap_or_else(|e| e.into_inner()); + txids + .iter() + .copied() + .filter(|txid| mempool.txstore.get(txid).is_none()) + .collect() + }; + let mut txs_map = HashMap::new(); - for &txid in txids { - if self.txstore.get(&txid).is_some() { - continue; - } + let mut unavailable = Vec::new(); + for txid in missing { match daemon.getmempooltx(&txid) { Ok(tx) => { txs_map.insert(txid, tx); } - Err(e) => warn!("add_by_txids cannot find txid='{}': e='{}'", txid, e), + Err(e) => { + warn!("cannot find mempool txid='{}': e='{}'", txid, e); + unavailable.push(txid); + } } } - if txs_map.is_empty() { - return Ok(()); + + if !txs_map.is_empty() { + let mut mempool = mempool.write().unwrap_or_else(|e| e.into_inner()); + // update() may have indexed some of these while we were fetching + txs_map.retain(|txid, _| mempool.txstore.get(txid).is_none()); + if !txs_map.is_empty() { + mempool.add(txs_map)?; + } } - self.add(txs_map) + Ok(unavailable) } #[trace] @@ -529,7 +551,7 @@ impl Mempool { // coexist in the local view. update() itself is safe (evictions // are applied before additions, diffed against one consistent // bitcoind snapshot), but Query::broadcast_raw()/submit_package() - // inject transactions via add_by_txid(s) outside the sync loop - + // inject transactions via ensure_tx(s) outside the sync loop - // broadcasting a replacement while the original is still indexed // makes the later add() clobber the original's `edges` entry. // Evicting the original then finds its entry gone or foreign. @@ -690,7 +712,13 @@ impl Mempool { if !fetched_txs.is_empty() { let mut mempool = mempool.write().unwrap(); - mempool.add(fetched_txs)?; + // Query::broadcast_raw()/submit_package() may have indexed some of these via + // ensure_tx(s) since the snapshot above; adding them twice would duplicate + // their history and recent entries. + fetched_txs.retain(|txid, _| mempool.txstore.get(txid).is_none()); + if !fetched_txs.is_empty() { + mempool.add(fetched_txs)?; + } count .with_label_values(&["txs"]) diff --git a/src/new_index/query.rs b/src/new_index/query.rs index c517c4801..937da8054 100644 --- a/src/new_index/query.rs +++ b/src/new_index/query.rs @@ -80,11 +80,7 @@ impl Query { #[trace] pub fn broadcast_raw(&self, txhex: &str) -> Result { let txid = self.daemon.broadcast_raw(txhex)?; - let _ = self - .mempool - .write() - .unwrap_or_else(|e| e.into_inner()) - .add_by_txid(&self.daemon, txid); + let _ = Mempool::ensure_tx(&self.mempool, &self.daemon, txid); Ok(txid) } @@ -100,11 +96,7 @@ impl Query { // immediately (they read from the local mempool), mirroring broadcast_raw() above. let accepted_txids = result.accepted_txids(); if !accepted_txids.is_empty() { - let _ = self - .mempool - .write() - .unwrap_or_else(|e| e.into_inner()) - .add_by_txids(&self.daemon, &accepted_txids); + let _ = Mempool::ensure_txs(&self.mempool, &self.daemon, &accepted_txids); } Ok(result) } diff --git a/tests/common.rs b/tests/common.rs index a18cb778a..8249dd676 100644 --- a/tests/common.rs +++ b/tests/common.rs @@ -451,6 +451,23 @@ impl TestRunner { .transaction() .unwrap()) } + + /// Number of client-proxied daemon RPCs that completed with the given `result` label. + pub fn proxied_rpc_count(&self, result: &str) -> f64 { + self.metrics + .gather() + .iter() + .filter(|family| family.name() == "daemon_rpc_proxied") + .flat_map(|family| family.get_metric()) + .filter(|metric| { + metric + .get_label() + .iter() + .any(|label| label.name() == "result" && label.value() == result) + }) + .map(|metric| metric.get_counter().value()) + .sum() + } } pub fn init_rest_tester() -> Result<(rest::Handle, net::SocketAddr, TestRunner)> { diff --git a/tests/rest.rs b/tests/rest.rs index 91100cf9c..fdffec1b3 100644 --- a/tests/rest.rs +++ b/tests/rest.rs @@ -1367,6 +1367,67 @@ fn test_rest_package_updates_mempool() -> Result<()> { Ok(()) } +#[cfg(not(feature = "liquid"))] +#[test] +fn test_mempool_ensure_txs_concurrency() -> Result<()> { + use electrs::new_index::Mempool; + use std::sync::{mpsc, Arc}; + use std::thread; + use std::time::{Duration, Instant}; + + const TIMEOUT: Duration = Duration::from_secs(10); + + let (rest_handle, rest_addr, tester) = common::init_rest_tester().unwrap(); + + let addr = tester.newaddress()?; + let txid: Txid = tester + .node_client() + .call("sendtoaddress", &[addr.to_string().into(), 0.1.into()]) + .unwrap(); + + // While a reader holds the mempool lock, the daemon fetch must still complete. If the fetch + // ran under the write lock it could not start until the reader is dropped. + let proxied_before = tester.proxied_rpc_count("ok"); + let reader = tester.mempool().read().unwrap(); + assert!(reader.lookup_txn(&txid).is_none()); + + let mempool = Arc::clone(tester.mempool()); + let daemon = Arc::clone(tester.daemon()); + let handle = thread::spawn(move || Mempool::ensure_tx(&mempool, &daemon, txid)); + + let deadline = Instant::now() + TIMEOUT; + while tester.proxied_rpc_count("ok") <= proxied_before { + assert!( + Instant::now() < deadline, + "daemon fetch did not complete while a mempool reader was active" + ); + thread::sleep(Duration::from_millis(10)); + } + drop(reader); + handle.join().unwrap().expect("ensure_tx failed"); + + assert!(tester.mempool().read().unwrap().lookup_txn(&txid).is_some()); + let res = get_json(rest_addr, &format!("/tx/{}", txid))?; + assert_eq!(res["status"]["confirmed"].as_bool(), Some(false)); + + // An already-indexed txid must return without the write lock or a daemon fetch, so it + // completes while another thread holds a reader. + let proxied_before = tester.proxied_rpc_count("ok"); + let reader = tester.mempool().read().unwrap(); + let (done_tx, done_rx) = mpsc::channel(); + let mempool = Arc::clone(tester.mempool()); + let daemon = Arc::clone(tester.daemon()); + thread::spawn(move || done_tx.send(Mempool::ensure_tx(&mempool, &daemon, txid))); + done_rx + .recv_timeout(TIMEOUT) + .expect("ensure_tx for an indexed txid blocked on the write lock")?; + drop(reader); + assert_eq!(tester.proxied_rpc_count("ok"), proxied_before); + + rest_handle.stop(); + Ok(()) +} + // Elements-only tests #[cfg(feature = "liquid")] @@ -1580,7 +1641,7 @@ fn test_rest_mempool_rbf_eviction() -> Result<()> { // Regression test for the mempool eviction panic ("missing mempool edge // for outpoint"): tx A and its RBF replacement B spend the same outpoint // and transiently coexist in the local mempool view when B is injected - // through the broadcast endpoint (add_by_txid) while A is still indexed. + // through the broadcast endpoint (ensure_tx) while A is still indexed. // B's add() clobbers A's `edges` entry; the next sync round evicts A and // must tolerate the missing/foreign edge instead of panicking. let (rest_handle, rest_addr, mut tester) = common::init_rest_tester().unwrap(); @@ -1618,7 +1679,7 @@ fn test_rest_mempool_rbf_eviction() -> Result<()> { let b_hex = finalized["hex"].as_str().expect("finalized tx hex"); // Inject B through the electrs broadcast endpoint: the node accepts the - // replacement (evicting A node-side), and add_by_txid() indexes B locally + // replacement (evicting A node-side), and ensure_tx() indexes B locally // while A is still present - clobbering A's edges entry for the shared // outpoint. let broadcast_resp = ureq::post(&format!("http://{}/tx", rest_addr)).send(b_hex)?; From da60cec1b60dc1b776562b24e5f54b8460593332 Mon Sep 17 00:00:00 2001 From: Oleksandr Fedorenko Date: Mon, 28 Sep 2026 15:11:58 +0300 Subject: [PATCH 22/28] harden mempool sync and daemon reconnect against transient failures - stop killing the process on transient mempool sync errors - back off between retry_reconnect attempts - replace the overloaded bool mempool sync result with an enum - stop pausing the main sync loop on mempool add failures - fix off-by-one in the mempool sync retry cap - add mempool_sync_failed_to_index metric - cover reconnect backoff and interruption in tests --- src/bin/electrs.rs | 56 ++++++++++++++++------- src/daemon.rs | 93 +++++++++++++++++++++++++++++++++------ src/electrum/server.rs | 2 +- src/new_index/mempool.rs | 41 +++++++++++++---- src/new_index/mod.rs | 2 +- src/rest.rs | 25 ++++++++--- src/rest/mempool_tests.rs | 5 ++- tests/common.rs | 12 +++-- 8 files changed, 186 insertions(+), 50 deletions(-) diff --git a/src/bin/electrs.rs b/src/bin/electrs.rs index 6f94d028b..8701b5393 100644 --- a/src/bin/electrs.rs +++ b/src/bin/electrs.rs @@ -8,7 +8,7 @@ extern crate log; extern crate electrs; use crossbeam_channel::{self as channel}; -use error_chain::ChainedError; +use error_chain::{bail, ChainedError}; use std::{env, process, thread}; use std::sync::{Arc, RwLock}; use std::time::Duration; @@ -20,7 +20,9 @@ use electrs::{ electrum::RPC as ElectrumRPC, errors::*, metrics::Metrics, - new_index::{precache, zmq, ChainQuery, FetchFrom, Indexer, Mempool, Query, Store}, + new_index::{ + precache, zmq, ChainQuery, FetchFrom, Indexer, Mempool, MempoolSyncStatus, Query, Store, + }, rest, signal::Waiter, }; @@ -35,6 +37,10 @@ use electrs::metrics::MetricOpts; /// Default salt rotation interval in seconds (24 hours) const DEFAULT_SALT_ROTATION_INTERVAL_SECS: u64 = 24 * 3600; +/// Maximum number of `FailedToIndex` retries during the initial mempool +/// sync before giving up. +const MAX_INITIAL_MEMPOOL_SYNC_RETRIES: u32 = 100; + fn fetch_from(config: &Config, store: &Store) -> FetchFrom { let mut jsonrpc_import = config.jsonrpc_import; if !jsonrpc_import { @@ -122,18 +128,32 @@ fn run_server(config: Arc, salt_rwlock: Arc>) -> Result<( Arc::clone(&config), ))); - // A mempool update is retried whenever the tip moves, so index and try again. An update - // that cannot advance the tip never will here, and no listener is bound yet. - while !Mempool::update(&mempool, &daemon, &tip)? { - let new_tip = indexer.update(&daemon)?; - if new_tip == tip { - warn!( - "index could not advance, starting up with a partial index tip='{}'", - tip - ); - break; + let mut mempool_sync_retries = 0; + loop { + match Mempool::update(&mempool, &daemon, &tip)? { + MempoolSyncStatus::Synced => break, + MempoolSyncStatus::TipMoved => { + let new_tip = indexer.update(&daemon)?; + if new_tip == tip { + warn!( + "index could not advance, starting up with a partial index tip='{}'", + tip + ); + break; + } + tip = new_tip; + } + MempoolSyncStatus::FailedToIndex => { + mempool_sync_retries += 1; + if mempool_sync_retries >= MAX_INITIAL_MEMPOOL_SYNC_RETRIES { + bail!( + "initial mempool sync failed to index {} times, giving up", + MAX_INITIAL_MEMPOOL_SYNC_RETRIES + ); + } + signal.wait(Duration::from_secs(3), false)?; + } } - tip = new_tip; } #[cfg(feature = "liquid")] @@ -185,8 +205,14 @@ fn run_server(config: Arc, salt_rwlock: Arc>) -> Result<( }; // Update mempool - if !Mempool::update(&mempool, &daemon, &tip)? { - warn!("skipped failed mempool update, trying again in 5 seconds"); + match Mempool::update(&mempool, &daemon, &tip)? { + MempoolSyncStatus::Synced => {} + MempoolSyncStatus::TipMoved => { + warn!("mempool sync aborted: chain tip moved, retrying next cycle"); + } + MempoolSyncStatus::FailedToIndex => { + warn!("mempool_sync_failed_to_index: retrying next cycle"); + } } // Update subscribed clients diff --git a/src/daemon.rs b/src/daemon.rs index d60b133aa..6feb7f231 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -87,8 +87,25 @@ lazy_static! { const MAX_ATTEMPTS: u32 = 5; const RETRY_WAIT_DURATION: Duration = Duration::from_secs(1); +const RECONNECT_RETRY_WAIT_DURATION: Duration = Duration::from_secs(3); const BLOCK_TEMPLATE_RPC_TIMEOUT: Duration = Duration::from_secs(30); +fn retry_reconnect_with_backoff(mut reconnect: F, mut wait: W) -> Result +where + F: FnMut() -> Result, + W: FnMut(Duration) -> Result<()>, +{ + loop { + match reconnect() { + Ok(value) => return Ok(value), + Err(e) => { + warn!("failed connecting to RPC daemon: {}", e.display_chain()); + wait(RECONNECT_RETRY_WAIT_DURATION)?; + } + } + } +} + #[trace] fn parse_hash(value: &Value) -> Result where @@ -1181,16 +1198,11 @@ impl Daemon { } #[trace] - fn retry_reconnect(&self) -> Daemon { - // XXX add a max reconnection attempts limit? - loop { - match self.reconnect() { - Ok(daemon) => break daemon, - Err(e) => { - warn!("failed connecting to RPC daemon: {}", e.display_chain()); - } - } - } + fn retry_reconnect(&self) -> Result { + retry_reconnect_with_backoff( + || self.reconnect(), + |duration| self.signal.wait(duration, false), + ) } // Send requests in parallel over multiple RPC connections as individual JSON-RPC requests (with no JSON-RPC batching), @@ -1219,9 +1231,15 @@ impl Daemon { // get initialized as necessary for the `rpc_threads` pool thread managed by rayon. thread_local!(static DAEMON_INSTANCE: OnceCell = OnceCell::new()); - DAEMON_INSTANCE.with(|daemon| { + DAEMON_INSTANCE.with(|daemon| -> Result { + if daemon.get().is_none() { + daemon + .set(self.retry_reconnect()?) + .unwrap_or_else(|_| unreachable!("thread-local cell was just empty")); + } daemon - .get_or_init(|| self.retry_reconnect()) + .get() + .expect("just initialized above") .retry_request(&method, ¶ms) }) }) @@ -1599,8 +1617,9 @@ impl Daemon { #[cfg(test)] mod tests { use super::{ - parse_jsonrpc_reply, recycle_due, BlockingSemaphore, Connection, ConnectionConfig, - CookieGetter, DAEMON_MAX_BODY_BYTES, DAEMON_MAX_HEADER_LINE_BYTES, + parse_jsonrpc_reply, recycle_due, retry_reconnect_with_backoff, BlockingSemaphore, + Connection, ConnectionConfig, CookieGetter, DAEMON_MAX_BODY_BYTES, + DAEMON_MAX_HEADER_LINE_BYTES, }; use crate::errors::{Error, ErrorKind, Result}; use crate::signal::Waiter; @@ -1622,6 +1641,52 @@ mod tests { Duration::from_millis(n) } + #[test] + fn reconnect_failures_are_backed_off_before_retrying() { + let mut attempts = 0; + let mut waits = vec![]; + + let result = retry_reconnect_with_backoff( + || { + attempts += 1; + if attempts < 3 { + bail!("post-connect setup failed"); + } + Ok(42) + }, + |duration| { + waits.push(duration); + Ok(()) + }, + ) + .unwrap(); + + assert_eq!(result, 42); + assert_eq!(attempts, 3); + assert_eq!(waits, vec![secs(3), secs(3)]); + } + + #[test] + fn reconnect_retry_stops_when_backoff_is_interrupted() { + let mut attempts = 0; + + let error = retry_reconnect_with_backoff::<(), _, _>( + || { + attempts += 1; + assert_eq!(attempts, 1, "reconnect retried after interrupted backoff"); + bail!("post-connect setup failed") + }, + |duration| { + assert_eq!(duration, secs(3)); + bail!("shutdown requested") + }, + ) + .unwrap_err(); + + assert_eq!(attempts, 1); + assert!(error.to_string().contains("shutdown requested")); + } + /// Spawn a fake daemon that runs `respond` against the accepted socket, and hand back a /// one-shot `Connection` pointed at it. The connection's I/O timeout doubles as its /// whole-request `recv` budget, so `io_timeout` is what the deadline tests turn on. diff --git a/src/electrum/server.rs b/src/electrum/server.rs index f79749504..30da5191e 100644 --- a/src/electrum/server.rs +++ b/src/electrum/server.rs @@ -416,7 +416,7 @@ impl Connection { let (chain_stats, mempool_stats) = self.query.stats(&script_hash[..])?; Ok(json!({ - "confirmed": chain_stats.funded_txo_sum - chain_stats.spent_txo_sum, + "confirmed": chain_stats.funded_txo_sum.saturating_sub(chain_stats.spent_txo_sum), "unconfirmed": mempool_stats.funded_txo_sum as i64 - mempool_stats.spent_txo_sum as i64, })) } diff --git a/src/new_index/mempool.rs b/src/new_index/mempool.rs index cf91763d8..fd71fdeec 100644 --- a/src/new_index/mempool.rs +++ b/src/new_index/mempool.rs @@ -1,4 +1,5 @@ use arraydeque::{ArrayDeque, Wrapping}; +use error_chain::ChainedError; use itertools::{Either, Itertools}; #[cfg(not(feature = "liquid"))] @@ -16,7 +17,7 @@ use crate::chain::{deserialize, BlockHash, Network, OutPoint, Transaction, TxOut use crate::config::Config; use crate::daemon::Daemon; use crate::errors::*; -use crate::metrics::{GaugeVec, HistogramOpts, HistogramVec, MetricOpts, Metrics}; +use crate::metrics::{Counter, GaugeVec, HistogramOpts, HistogramVec, MetricOpts, Metrics}; use crate::new_index::{ compute_script_hash, schema::FullHash, ChainQuery, FundingInfo, GetAmountVal, ScriptStats, SpendingInfo, SpendingInput, TxHistoryInfo, Utxo, @@ -30,6 +31,13 @@ use crate::elements::asset; const RECENT_TXS_SIZE: usize = 10; const BACKLOG_STATS_TTL: u64 = 10; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MempoolSyncStatus { + Synced, + TipMoved, + FailedToIndex, +} + pub struct Mempool { chain: Arc, config: Arc, @@ -46,6 +54,7 @@ pub struct Mempool { latency: HistogramVec, // mempool requests latency delta: HistogramVec, // # of added/removed txs count: GaugeVec, // current state of the mempool + sync_failed_to_index: Counter, // # of times a mempool sync failed to index fetched txs // elements only #[cfg(feature = "liquid")] @@ -79,7 +88,9 @@ impl Mempool { recent: ArrayDeque::new(), backlog_stats: ( BacklogStats::default(), - Instant::now() - Duration::from_secs(BACKLOG_STATS_TTL), + Instant::now() + .checked_sub(Duration::from_secs(BACKLOG_STATS_TTL)) + .unwrap_or_else(Instant::now), ), latency: metrics.histogram_vec( HistogramOpts::new("mempool_latency", "Mempool requests latency (in seconds)"), @@ -93,6 +104,10 @@ impl Mempool { MetricOpts::new("mempool_count", "# of elements currently at the mempool"), &["type"], ), + sync_failed_to_index: metrics.counter(MetricOpts::new( + "mempool_sync_failed_to_index", + "# of times a mempool sync failed to index fetched transactions", + )), #[cfg(feature = "liquid")] asset_history: HashMap::new(), @@ -601,14 +616,13 @@ impl Mempool { .map_or_else(|| vec![], |entries| self._history(entries, limit)) } - /// Sync our local view of the mempool with the bitcoind Daemon RPC. If the chain tip moves before - /// the mempool is fetched in full, syncing is aborted and an Ok(false) is returned. + /// Sync our local view of the mempool with the bitcoind Daemon RPC. #[trace] pub fn update( mempool: &Arc>, daemon: &Daemon, tip: &BlockHash, - ) -> Result { + ) -> Result { let (_timer, count) = { let mempool = mempool.read().unwrap(); let timer = mempool.latency.with_label_values(&["update"]).start_timer(); @@ -653,7 +667,7 @@ impl Mempool { .set(new_txids.len() as f64); if new_txids.is_empty() { - return Ok(true); + return Ok(MempoolSyncStatus::Synced); } // Fetch missing transactions from bitcoind @@ -662,7 +676,7 @@ impl Mempool { // Abort if the chain tip moved while fetching transactions if daemon.getbestblockhash()? != *tip { warn!("chain tip moved while updating mempool"); - return Ok(false); + return Ok(MempoolSyncStatus::TipMoved); } // Find which transactions were requested but are no longer available in bitcoind's mempool, @@ -710,6 +724,7 @@ impl Mempool { // Add fetched transactions to our view of the mempool trace!("indexing {} new mempool transactions", fetched_txs.len()); if !fetched_txs.is_empty() { + let fetched_txs_len = fetched_txs.len(); let mut mempool = mempool.write().unwrap(); // Query::broadcast_raw()/submit_package() may have indexed some of these via @@ -717,7 +732,15 @@ impl Mempool { // their history and recent entries. fetched_txs.retain(|txid, _| mempool.txstore.get(txid).is_none()); if !fetched_txs.is_empty() { - mempool.add(fetched_txs)?; + if let Err(e) = mempool.add(fetched_txs) { + warn!( + "transient failure adding {} fetched txs, skipping until next cycle: {}", + fetched_txs_len, + e.display_chain() + ); + mempool.sync_failed_to_index.inc(); + return Ok(MempoolSyncStatus::FailedToIndex); + } } count @@ -732,7 +755,7 @@ impl Mempool { trace!("mempool is synced"); - Ok(true) + Ok(MempoolSyncStatus::Synced) } } diff --git a/src/new_index/mod.rs b/src/new_index/mod.rs index 8d2a734ff..66bd8ab8d 100644 --- a/src/new_index/mod.rs +++ b/src/new_index/mod.rs @@ -10,7 +10,7 @@ pub mod zmq; pub use self::db::{DBRow, DB}; pub use self::fetch::{BlockEntry, FetchFrom}; -pub use self::mempool::Mempool; +pub use self::mempool::{Mempool, MempoolSyncStatus}; pub use self::query::Query; pub use self::schema::{ compute_script_hash, parse_hash, ChainQuery, FundingInfo, GetAmountVal, Indexer, ScriptStats, diff --git a/src/rest.rs b/src/rest.rs index a1b272dff..ee7858eda 100644 --- a/src/rest.rs +++ b/src/rest.rs @@ -479,14 +479,18 @@ impl Default for SpendingValue { fn ttl_by_depth(height: Option, query: &Query) -> u32 { height.map_or(TTL_SHORT, |height| { - if query.chain().best_height() - height >= CONF_FINAL { - TTL_LONG - } else { - TTL_SHORT - } + ttl_by_confirmations(query.chain().best_height(), height) }) } +fn ttl_by_confirmations(best_height: usize, height: usize) -> u32 { + if best_height.saturating_sub(height) >= CONF_FINAL { + TTL_LONG + } else { + TTL_SHORT + } +} + // Own the mempool data needed by a response. Chain IO and response construction // happen after capture returns and releases the read guard. struct MempoolTxs { @@ -1778,13 +1782,22 @@ impl From for HttpError { #[cfg(test)] mod tests { - use crate::rest::{is_block_template_request, HttpError}; + use crate::rest::{ + is_block_template_request, ttl_by_confirmations, HttpError, TTL_LONG, TTL_SHORT, + }; use crate::{errors, errors::ErrorKind}; use http_body_util::BodyExt; use hyper::{Method, StatusCode}; use serde_json::Value; use std::collections::HashMap; + #[test] + fn ttl_by_confirmations_survives_a_reorg_race() { + assert_eq!(ttl_by_confirmations(110, 100), TTL_LONG); + assert_eq!(ttl_by_confirmations(105, 100), TTL_SHORT); + assert_eq!(ttl_by_confirmations(99, 100), TTL_SHORT); + } + #[test] fn block_template_is_the_only_route_kept_on_the_async_runtime() { let is_async = |method: Method, uri: &str| { diff --git a/src/rest/mempool_tests.rs b/src/rest/mempool_tests.rs index 051111b29..976c77591 100644 --- a/src/rest/mempool_tests.rs +++ b/src/rest/mempool_tests.rs @@ -94,7 +94,10 @@ fn prevout_lookup_survives_ancestor_eviction() -> Result<()> { .call("sendrawtransaction", &[signed_conflict["hex"].clone()])?; let tip = tester.get_best_block_hash()?; - assert!(Mempool::update(&tester.mempool(), &tester.daemon(), &tip)?); + assert_eq!( + Mempool::update(&tester.mempool(), &tester.daemon(), &tip)?, + crate::new_index::MempoolSyncStatus::Synced + ); // Exercise the actual REST preparation path after the ancestor is gone. // Keep a writer guard during preparation: no mempool re-read is allowed. diff --git a/tests/common.rs b/tests/common.rs index 8249dd676..ee985d077 100644 --- a/tests/common.rs +++ b/tests/common.rs @@ -24,7 +24,7 @@ use electrs::{ daemon::Daemon, electrum::RPC as ElectrumRPC, metrics::Metrics, - new_index::{ChainQuery, FetchFrom, Indexer, Mempool, Query, Store}, + new_index::{ChainQuery, FetchFrom, Indexer, Mempool, MempoolSyncStatus, Query, Store}, rest, signal::Waiter, }; @@ -202,7 +202,10 @@ impl TestRunner { &metrics, Arc::clone(&config), ))); - assert!(Mempool::update(&mempool, &daemon, &tip)?); + assert_eq!( + Mempool::update(&mempool, &daemon, &tip)?, + MempoolSyncStatus::Synced + ); let query = Arc::new(Query::new( Arc::clone(&chain), @@ -258,7 +261,10 @@ impl TestRunner { pub fn sync(&mut self) -> Result<()> { let tip = self.indexer.update(&self.daemon)?; - assert!(Mempool::update(&self.mempool, &self.daemon, &tip)?); + assert_eq!( + Mempool::update(&self.mempool, &self.daemon, &tip)?, + MempoolSyncStatus::Synced + ); // force an update for the mempool stats, which are normally cached self.mempool.write().unwrap().update_backlog_stats(); Ok(()) From 0231fff890f4e56745a3d2b7ccde3795e7a4bff0 Mon Sep 17 00:00:00 2001 From: Oleksandr Fedorenko Date: Wed, 30 Sep 2026 11:23:21 +0300 Subject: [PATCH 23/28] integration: route TooBigHistory through HTTP 400 after mempool-snapshot merge The prepare_history path from fix/rest-mempool-snapshot-race wraps chain lookups in HttpError::lookup (503); the scan-limit branch expects TooBigHistory to reach clients as 400. Classify it explicitly. --- src/rest.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/rest.rs b/src/rest.rs index ee7858eda..e8108ca09 100644 --- a/src/rest.rs +++ b/src/rest.rs @@ -1651,6 +1651,11 @@ impl HttpError { } fn lookup(err: errors::Error) -> Self { + // A history that exceeds the configured scan limit is a property of the + // requested scripthash, not a transient server-side failure. + if let errors::ErrorKind::TooBigHistory = err.kind() { + return HttpError::from(err.to_string()); + } let status = StatusCode::SERVICE_UNAVAILABLE; warn!("REST lookup failed status='{}' err='{:?}'", status, err); HttpError(status, err.to_string()) From 9412b5cfc3d98f53922fd4183384eaeb8bb009e5 Mon Sep 17 00:00:00 2001 From: Edward Houston Date: Wed, 30 Sep 2026 14:38:21 +0200 Subject: [PATCH 24/28] Harden scan limits, DB compatibility, mempool sync and Electrum response limits Bound utxo checkpoints with --utxos-checkpoint-limit, retry TooBigHistory in precache, stop cursor scans past the cursor height, and reject a zero history scan limit. Keep the V record readable by older binaries and store index flags under VF. Index broadcasts from the submitted hex, isolate unresolvable mempool txs and stop after repeated sync failures. Reserve rebuild memory in the checkpoint Merkle cache, require a write timeout or max age with a finite response budget, do not abort completed writes at the deadline, and fix the header range overflow and the Liquid MissingTxo txid parsing. --- README.md | 6 +- src/bin/electrs.rs | 19 +++++- src/config.rs | 55 ++++++++++++++-- src/electrum/client_io.rs | 46 +++++++++---- src/electrum/server.rs | 9 ++- src/new_index/db.rs | 91 +++++++++++++++++++------- src/new_index/mempool.rs | 87 +++++++++++++++++++++---- src/new_index/precache.rs | 19 +++++- src/new_index/query.rs | 31 +++++++-- src/new_index/schema.rs | 85 +++++++++++++++++++----- src/rest.rs | 11 +++- src/rest/mempool_tests.rs | 4 +- src/util/electrum_merkle.rs | 126 ++++++++++++++++++++++++++++++------ tests/common.rs | 1 + tests/config.rs | 68 +++++++++++++++++++ tests/scan_limits.rs | 87 +++++++++++++++++++++++++ 16 files changed, 639 insertions(+), 106 deletions(-) diff --git a/README.md b/README.md index e53c71f4b..03cc016ca 100644 --- a/README.md +++ b/README.md @@ -72,10 +72,12 @@ In addition to electrs's original configuration options, a few new options are a - `--index-unspendables` - enables indexing of provably unspendable outputs. - `--enable-mining-rest` - enables cached mining-related HTTP endpoints. - `--utxos-limit ` - maximum number of utxos to return per address. +- `--history-scan-limit ` - approximate number of history rows scanned per address lookup before failing with a "too popular" error; progress is checkpointed so repeated lookups make headway. Must be at least `1`. Default: `100000`. +- `--utxos-checkpoint-limit ` - maximum number of utxos held in a partially scanned utxo set and its saved checkpoint, and of unconfirmed history entries scanned per address. Must be at least `--utxos-limit`. Default: 10 times `--utxos-limit`. - `--electrum-txs-limit ` - maximum number of txs to return per address in the electrum server (does not apply for the http api). -- `--electrum-rpc-write-timeout ` - maximum time to transmit a complete Electrum reply, including a whole batch and its newline, an error reply, or a subscription notification. The deadline starts at the first write and is not extended by partial progress. Expiry closes the connection and releases its response buffers. Idle subscriptions and command execution are unaffected. Default: `30`, matching the RPC proxy's `CLIENT_WRITE_TIMEOUT_SECS`; `0` disables the bound. The `electrum_client_write_timeouts_total` counter records connections closed by this deadline. +- `--electrum-rpc-write-timeout ` - maximum time to transmit a complete Electrum reply, including a whole batch and its newline, an error reply, or a subscription notification. The deadline starts at the first write and is not extended by partial progress. Expiry closes the connection and releases its response buffers. Idle subscriptions and command execution are unaffected. Default: `30`, matching the RPC proxy's `CLIENT_WRITE_TIMEOUT_SECS`; `0` disables the bound, which is only accepted with `--electrum-rpc-conn-max-age` set or the global response budget disabled. The `electrum_client_write_timeouts_total` counter records connections closed by this deadline. - `--electrum-rpc-max-response-num-bytes ` - maximum size, in bytes, of a single Electrum solicited reply line (brackets, commas, error objects, and newline included). Overflowing replies return a correlated error (code 1) and the rest of the batch is not executed; the command that overflowed may already have completed. Subscription notifications are exempt. `0` disables the cap. Default: `8388608` (8 MiB) on Bitcoin, `33554432` (32 MiB) on Liquid (sized for a 2016-header window during a dynafed parameter vote). -- `--electrum-rpc-global-response-budget-bytes ` - aggregate cap, in bytes, on Electrum response-buffer memory retained across all connections at any instant. Small replies below a 16 KiB per-connection floor are exempt; above the floor each reply charges the budget in 64 KiB chunks (or a smaller final chunk) and releases on drop. Requests that would exceed the budget are rejected with a server error (code 2) instead of allocating, and the connection stays usable. Must be `>= --electrum-rpc-max-response-num-bytes`. `0` disables the budget. Default: `67108864` (64 MiB) on Bitcoin, `268435456` (256 MiB) on Liquid. +- `--electrum-rpc-global-response-budget-bytes ` - aggregate cap, in bytes, on Electrum solicited-reply buffer memory retained across all connections at any instant, including batch replies held while later batch elements execute. Subscription notifications and queued request lines are not counted. Small replies below a 16 KiB per-connection floor are exempt; above the floor each reply charges the budget in 64 KiB chunks (or a smaller final chunk) and releases on drop. Requests that would exceed the budget are rejected with a server error (code 2) instead of allocating, and the connection stays usable. Must be `>= --electrum-rpc-max-response-num-bytes`, and requires a non-zero `--electrum-rpc-write-timeout` or `--electrum-rpc-conn-max-age`. `0` disables the budget. Default: `67108864` (64 MiB) on Bitcoin, `268435456` (256 MiB) on Liquid. - `--electrum-banner ` - welcome banner text for electrum server. Additional options with the `liquid` feature: diff --git a/src/bin/electrs.rs b/src/bin/electrs.rs index 8701b5393..b051a7b1b 100644 --- a/src/bin/electrs.rs +++ b/src/bin/electrs.rs @@ -41,6 +41,10 @@ const DEFAULT_SALT_ROTATION_INTERVAL_SECS: u64 = 24 * 3600; /// sync before giving up. const MAX_INITIAL_MEMPOOL_SYNC_RETRIES: u32 = 100; +/// Maximum number of consecutive `FailedToIndex` main loop cycles before giving up, +/// about an hour at the 5 second loop interval. +const MAX_CONSECUTIVE_MEMPOOL_SYNC_FAILURES: u32 = 720; + fn fetch_from(config: &Config, store: &Store) -> FetchFrom { let mut jsonrpc_import = config.jsonrpc_import; if !jsonrpc_import { @@ -188,6 +192,7 @@ fn run_server(config: Arc, salt_rwlock: Arc>) -> Result<( "count of iterations of electrs main loop each 5 seconds or after interrupts", )); + let mut mempool_sync_failures = 0; loop { main_loop_count.inc(); @@ -206,12 +211,22 @@ fn run_server(config: Arc, salt_rwlock: Arc>) -> Result<( // Update mempool match Mempool::update(&mempool, &daemon, &tip)? { - MempoolSyncStatus::Synced => {} + MempoolSyncStatus::Synced => mempool_sync_failures = 0, MempoolSyncStatus::TipMoved => { warn!("mempool sync aborted: chain tip moved, retrying next cycle"); } MempoolSyncStatus::FailedToIndex => { - warn!("mempool_sync_failed_to_index: retrying next cycle"); + mempool_sync_failures += 1; + if mempool_sync_failures >= MAX_CONSECUTIVE_MEMPOOL_SYNC_FAILURES { + bail!( + "mempool sync failed to index {} consecutive times, giving up", + MAX_CONSECUTIVE_MEMPOOL_SYNC_FAILURES + ); + } + warn!( + "mempool_sync_failed_to_index: retrying next cycle failures='{}'", + mempool_sync_failures + ); } } diff --git a/src/config.rs b/src/config.rs index bc82e354e..91491aee8 100644 --- a/src/config.rs +++ b/src/config.rs @@ -76,6 +76,7 @@ pub struct Config { pub precache_scripts: Option, pub utxos_limit: usize, pub history_scan_limit: usize, + pub utxos_checkpoint_limit: usize, pub electrum_txs_limit: usize, pub electrum_subscription_limit: usize, pub electrum_checkpoint_proof_concurrency_limit: usize, @@ -221,7 +222,7 @@ impl Config { .arg( Arg::with_name("electrum_rpc_write_timeout") .long("electrum-rpc-write-timeout") - .help("Maximum time (in seconds) to transmit a complete Electrum response, including a whole batch, errors, or a notification. Starts at the first write and is not extended by partial progress. Closes stalled connections and releases response buffers. Does not time idle subscriptions or command execution. 0 = disabled (default: 30).") + .help("Maximum time (in seconds) to transmit a complete Electrum response, including a whole batch, errors, or a notification. Starts at the first write and is not extended by partial progress. Closes stalled connections and releases response buffers. Does not time idle subscriptions or command execution. 0 = disabled, which requires --electrum-rpc-conn-max-age or an unlimited --electrum-rpc-global-response-budget-bytes (default: 30).") .default_value("30") .takes_value(true), ) @@ -241,7 +242,7 @@ impl Config { .arg( Arg::with_name("electrum_rpc_global_response_budget_bytes") .long("electrum-rpc-global-response-budget-bytes") - .help("Aggregate cap (in bytes) on Electrum response-buffer memory retained across all connections at any instant. Bounds the memory a set of non-reading clients can hold while writer threads block in write_all. Small replies (< 16 KiB per connection) are exempt; above that floor each reply charges the budget in 64 KiB chunks (or a smaller final chunk) and releases on drop. Requests that would exceed the budget are rejected with a server error (code 2) instead of allocating. Must be >= --electrum-rpc-max-response-num-bytes so a single maximum-size reply can succeed. 0 = unlimited (default: 67108864, i.e. 64 MiB on Bitcoin; 268435456, i.e. 256 MiB on Liquid, keeping the same ratio to the per-line cap).") + .help("Aggregate cap (in bytes) on Electrum solicited-reply buffer memory retained across all connections at any instant, including batch replies held while later batch elements execute and replies held while writer threads block on non-reading clients. Subscription notifications and queued request lines are not counted. Small replies (< 16 KiB per connection) are exempt; above that floor each reply charges the budget in 64 KiB chunks (or a smaller final chunk) and releases on drop. Requests that would exceed the budget are rejected with a server error (code 2) instead of allocating. Must be >= --electrum-rpc-max-response-num-bytes so a single maximum-size reply can succeed, and requires a non-zero --electrum-rpc-write-timeout or --electrum-rpc-conn-max-age so stalled clients release their share. 0 = unlimited (default: 67108864, i.e. 64 MiB on Bitcoin; 268435456, i.e. 256 MiB on Liquid, keeping the same ratio to the per-line cap).") .takes_value(true), ) .arg( @@ -327,9 +328,15 @@ impl Config { .arg( Arg::with_name("history_scan_limit") .long("history-scan-limit") - .help("Approximate number of scripthash history rows scanned per utxo/stats lookup before giving up with a \"too popular\" error (the scan reads through the end of the height it is on when the limit is reached, so it may process somewhat more). Unlike --utxos-limit and --electrum-txs-limit, which cap the size of the result, this bounds the amount of work done per request regardless of the outcome.") + .help("Approximate number of scripthash history rows scanned per utxo/stats lookup before giving up with a \"too popular\" error (the scan reads through the end of the height it is on when the limit is reached, so it may process somewhat more). Unlike --utxos-limit and --electrum-txs-limit, which cap the size of the result, this bounds the amount of work done per request regardless of the outcome. Must be at least 1.") .default_value("100000") ) + .arg( + Arg::with_name("utxos_checkpoint_limit") + .long("utxos-checkpoint-limit") + .help("Maximum number of utxos held in a partially scanned utxo set, and in the checkpoint saved for it. An address whose utxo set grows past this while scanning fails with a \"too popular\" error and no checkpoint is saved. Also bounds the unconfirmed history entries scanned per address for utxo and stats lookups. Must be at least --utxos-limit (default: 10 times --utxos-limit).") + .takes_value(true) + ) .arg( Arg::with_name("electrum_txs_limit") .long("electrum-txs-limit") @@ -349,7 +356,7 @@ impl Config { ).arg( Arg::with_name("electrum_checkpoint_merkle_cache_mb") .long("electrum-checkpoint-merkle-cache-mb") - .help("Approximate memory budget, in MB, for checkpoint Merkle proof handling: cached trees and the working memory of concurrent rebuilds share this one budget (independently of electrum-checkpoint-proof-concurrency-limit, so a high core count can no longer multiply memory use past it). It is a soft target based on an estimate, not a hard ceiling. Entry size scales with cp_height (up to ~28MB near the chain tip). A cp_height too large to ever fit is served without being cached.") + .help("Approximate memory budget, in MB, for checkpoint Merkle proof handling: cached trees and the working memory of concurrent rebuilds share this one budget (independently of electrum-checkpoint-proof-concurrency-limit, so a high core count can no longer multiply memory use past it). Cached trees use at most half of it, so a full cache still leaves room for rebuilds. It is a soft target based on an estimate, not a hard ceiling. Entry size scales with cp_height (up to ~28MB near the chain tip). A cp_height too large to ever fit is served without being cached.") .default_value("256") .takes_value(true) ).arg( @@ -619,6 +626,41 @@ impl Config { ) .exit(); } + if electrum_rpc_global_response_budget_bytes != usize::MAX + && electrum_rpc_write_timeout.is_none() + && electrum_rpc_conn_max_age.is_none() + { + clap::Error::with_description( + "--electrum-rpc-global-response-budget-bytes requires a non-zero \ + --electrum-rpc-write-timeout or --electrum-rpc-conn-max-age; otherwise \ + clients that stop reading hold their share of the budget until restart", + clap::ErrorKind::ValueValidation, + ) + .exit(); + } + let utxos_limit = value_t_or_exit!(m, "utxos_limit", usize); + let history_scan_limit = value_t_or_exit!(m, "history_scan_limit", usize); + if history_scan_limit == 0 { + clap::Error::with_description( + "--history-scan-limit must be at least 1", + clap::ErrorKind::ValueValidation, + ) + .exit(); + } + let utxos_checkpoint_limit = match m.value_of("utxos_checkpoint_limit") { + Some(_) => value_t_or_exit!(m, "utxos_checkpoint_limit", usize), + None => utxos_limit.saturating_mul(10), + }; + if utxos_checkpoint_limit < utxos_limit { + clap::Error::with_description( + &format!( + "--utxos-checkpoint-limit ({}) must be >= --utxos-limit ({})", + utxos_checkpoint_limit, utxos_limit + ), + clap::ErrorKind::ValueValidation, + ) + .exit(); + } let http_addr: SocketAddr = str_to_socketaddr( m.value_of("http_addr") .unwrap_or(&format!("127.0.0.1:{}", default_http_port)), @@ -687,8 +729,9 @@ impl Config { daemon_parallelism: value_t_or_exit!(m, "daemon_parallelism", usize), daemon_conn_max_age, cookie, - utxos_limit: value_t_or_exit!(m, "utxos_limit", usize), - history_scan_limit: value_t_or_exit!(m, "history_scan_limit", usize), + utxos_limit, + history_scan_limit, + utxos_checkpoint_limit, electrum_rpc_addr, electrum_rpc_conn_max_age, electrum_rpc_write_timeout, diff --git a/src/electrum/client_io.rs b/src/electrum/client_io.rs index 5a4528dfe..b62ce4278 100644 --- a/src/electrum/client_io.rs +++ b/src/electrum/client_io.rs @@ -96,21 +96,21 @@ impl Write for ClientWriter<'_> { // A fresh full timeout on each partial write would let a trickling // reader extend the response indefinitely. Only use the remaining time. self.stream.set_write_timeout(Some(deadline - now))?; - let result = (&*self.stream).write(bytes); - if Instant::now() >= deadline - || matches!(&result, Err(error) if matches!(error.kind(), io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut)) - { - return Err(self.expired()); + match (&*self.stream).write(bytes) { + Err(error) + if matches!( + error.kind(), + io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut + ) => + { + Err(self.expired()) + } + result => result, } - result } fn flush(&mut self) -> io::Result<()> { - if self.timed_out - || self - .deadline - .map_or(false, |deadline| Instant::now() >= deadline) - { + if self.timed_out { return Err(self.expired()); } Ok(()) @@ -280,6 +280,30 @@ mod tests { assert_eq!(timeouts.get(), 1); } + #[test] + fn write_accepted_by_the_kernel_is_not_aborted_at_the_deadline() { + let sockets = SocketPair::new(); + let timeouts = counter(); + let result = write_to_client( + &sockets.server, + Some(Duration::from_millis(50)), + &timeouts, + |writer| { + writer + .write_all(b"payload") + .chain_err(|| "payload failed")?; + thread::sleep(Duration::from_millis(100)); + writer.flush().chain_err(|| "flush failed") + }, + ); + assert!(result.is_ok(), "completed write was treated as a timeout"); + assert_eq!(timeouts.get(), 0); + let mut client = sockets.client.try_clone().unwrap(); + let mut bytes = [0; 7]; + client.read_exact(&mut bytes).unwrap(); + assert_eq!(&bytes, b"payload"); + } + #[test] fn partial_progress_does_not_extend_the_deadline() { let sockets = SocketPair::new(); diff --git a/src/electrum/server.rs b/src/electrum/server.rs index 30da5191e..7115015e4 100644 --- a/src/electrum/server.rs +++ b/src/electrum/server.rs @@ -316,8 +316,11 @@ impl Connection { let start_height = usize_from_value(params.get(0), "start_height")?; let count = MAX_HEADERS.min(usize_from_value(params.get(1), "count")?); let cp_height = usize_from_value_or(params.get(2), "cp_height", 0)?; + let end_height = start_height + .checked_add(count) + .chain_err(|| "start_height + count overflows")?; if count == 0 || cp_height == 0 { - let headers: Vec = (start_height..(start_height + count)) + let headers: Vec = (start_height..end_height) .filter_map(|height| { self.query .chain() @@ -334,7 +337,7 @@ impl Connection { let (branch, root, proof_cp_hash) = get_header_merkle_proof( self.query.chain(), - start_height + (count - 1), + end_height - 1, cp_height, self.checkpoint_proof_concurrency_limit, )?; @@ -346,7 +349,7 @@ impl Connection { == Some(&proof_cp_hash), "chain changed while creating checkpoint merkle proof" ); - let headers: Vec = (start_height..(start_height + count)) + let headers: Vec = (start_height..end_height) .filter_map(|height| { indexed_headers .header_by_height(height) diff --git a/src/new_index/db.rs b/src/new_index/db.rs index e43d7a5f2..9cac4e8df 100644 --- a/src/new_index/db.rs +++ b/src/new_index/db.rs @@ -380,33 +380,54 @@ impl DB { } fn verify_compatibility(&self, config: &Config) { - let compatibility_bytes = bincode::serialize_little(&( - DB_VERSION, - config.light_mode, - config.index_unspendables, - config.address_search, - )) - .unwrap(); - - let stored = match self.get(b"V") { - None => return self.put(b"V", &compatibility_bytes), - Some(stored) if stored == compatibility_bytes => return, + let version_bytes = bincode::serialize_little(&(DB_VERSION, config.light_mode)).unwrap(); + let flags_bytes = + bincode::serialize_little(&(config.index_unspendables, config.address_search)) + .unwrap(); + // Kept apart from "V" so binaries that only know (DB_VERSION, light_mode) can still open the DB. + const FLAGS_KEY: &[u8] = b"VF"; + + let stored_version = match self.get(b"V") { + None => { + self.put(FLAGS_KEY, &flags_bytes); + return self.put(b"V", &version_bytes); + } Some(stored) => stored, }; - let legacy_bytes = bincode::serialize_little(&(DB_VERSION, config.light_mode)).unwrap(); - if stored == legacy_bytes { - warn!( - "upgrading legacy DB compatibility record using index_unspendables={} and address_search={}; \ - these values must match the settings originally used to build this database, and \ - historical values cannot be verified", + if stored_version != version_bytes { + let combined_bytes = bincode::serialize_little(&( + DB_VERSION, + config.light_mode, config.index_unspendables, config.address_search, - ); - return self.put(b"V", &compatibility_bytes); + )) + .unwrap(); + if stored_version != combined_bytes { + panic!("Incompatible database found. Please reindex or migrate.") + } + self.put(FLAGS_KEY, &flags_bytes); + return self.put(b"V", &version_bytes); } - panic!("Incompatible database found. Please reindex or migrate.") + match self.get(FLAGS_KEY) { + Some(stored_flags) if stored_flags == flags_bytes => {} + Some(_) => panic!( + "Incompatible database found: index_unspendables={} and address_search={} \ + differ from the settings used to build this database. Please reindex.", + config.index_unspendables, config.address_search, + ), + None => { + warn!( + "recording DB index flags index_unspendables={} and address_search={}; \ + these values must match the settings originally used to build this database, and \ + historical values cannot be verified", + config.index_unspendables, + config.address_search, + ); + self.put(FLAGS_KEY, &flags_bytes); + } + } } #[cfg(test)] @@ -658,6 +679,7 @@ mod tests { precache_scripts: None, utxos_limit: 100, history_scan_limit: 100_000, + utxos_checkpoint_limit: 1_000, electrum_txs_limit: 100, electrum_subscription_limit: 10_000, electrum_checkpoint_proof_concurrency_limit: 2, @@ -732,14 +754,37 @@ mod tests { let config = make_test_config(true, true, false); db.verify_compatibility(&config); - let expected = bincode::serialize_little(&(DB_VERSION, false, true, true)).unwrap(); - assert_eq!(db.get(b"V").unwrap(), expected); + assert_eq!( + db.get(b"V").unwrap(), + legacy_bytes, + "the version record must stay readable by binaries that predate the flags" + ); let toggled = make_test_config(false, true, false); let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { db.verify_compatibility(&toggled); })); - assert!(result.is_err(), "upgraded record should still guard future mismatches"); + assert!(result.is_err(), "recorded flags should still guard future mismatches"); + } + + #[test] + fn test_verify_compatibility_restores_legacy_record_from_combined_format() { + let dir = tempfile::tempdir().unwrap(); + let db = DB::open_test(dir.path()); + + let combined = bincode::serialize_little(&(DB_VERSION, false, true, false)).unwrap(); + db.put(b"V", &combined); + + db.verify_compatibility(&make_test_config(true, false, false)); + + let legacy_bytes = bincode::serialize_little(&(DB_VERSION, false)).unwrap(); + assert_eq!(db.get(b"V").unwrap(), legacy_bytes); + + let toggled = make_test_config(true, true, false); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + db.verify_compatibility(&toggled); + })); + assert!(result.is_err(), "flags from the combined record must be carried over"); } #[test] diff --git a/src/new_index/mempool.rs b/src/new_index/mempool.rs index fd71fdeec..22efa964c 100644 --- a/src/new_index/mempool.rs +++ b/src/new_index/mempool.rs @@ -201,7 +201,7 @@ impl Mempool { Some(entries) => entries, }; ensure!( - entries.len() <= self.config.utxos_limit, + entries.len() <= self.config.utxos_checkpoint_limit, ErrorKind::TooBigHistory ); @@ -252,7 +252,7 @@ impl Mempool { Some(entries) => entries, }; ensure!( - entries.len() <= self.config.utxos_limit, + entries.len() <= self.config.utxos_checkpoint_limit, ErrorKind::TooBigHistory ); @@ -334,17 +334,33 @@ impl Mempool { /// already indexed and cannot be fetched from the daemon. #[trace] pub fn ensure_tx(mempool: &RwLock, daemon: &Daemon, txid: Txid) -> Result<()> { - if !Self::add_missing(mempool, daemon, &[txid])?.is_empty() { - bail!("ensure_tx cannot find txid='{}'", txid); - } - Ok(()) + Self::ensure_txs_with(mempool, daemon, &[txid], HashMap::new()) } /// Ensure multiple transactions (e.g. an accepted package) are in the mempool in a single batch, - /// so that interdependent parent/child txs are linked within the same `add` call. Txids that - /// are already present or cannot be fetched from the daemon are skipped. + /// so that interdependent parent/child txs are linked within the same `add` call. pub fn ensure_txs(mempool: &RwLock, daemon: &Daemon, txids: &[Txid]) -> Result<()> { - Self::add_missing(mempool, daemon, txids).map(|_| ()) + Self::ensure_txs_with(mempool, daemon, txids, HashMap::new()) + } + + /// Like `ensure_txs`, but takes the transactions found in `known` as-is instead of fetching + /// them from the daemon. Fails if any txid could not be fetched or indexed. + pub fn ensure_txs_with( + mempool: &RwLock, + daemon: &Daemon, + txids: &[Txid], + known: HashMap, + ) -> Result<()> { + let unavailable = Self::add_missing(mempool, daemon, txids, known)?; + if !unavailable.is_empty() { + bail!( + "cannot find {} of {} mempool txids first='{}'", + unavailable.len(), + txids.len(), + unavailable[0] + ); + } + Ok(()) } /// Fetch and index the given txids that are not yet in the local mempool, returning the ones @@ -354,6 +370,7 @@ impl Mempool { mempool: &RwLock, daemon: &Daemon, txids: &[Txid], + mut known: HashMap, ) -> Result> { let missing: Vec = { let mempool = mempool.read().unwrap_or_else(|e| e.into_inner()); @@ -367,6 +384,10 @@ impl Mempool { let mut txs_map = HashMap::new(); let mut unavailable = Vec::new(); for txid in missing { + if let Some(tx) = known.remove(&txid) { + txs_map.insert(txid, tx); + continue; + } match daemon.getmempooltx(&txid) { Ok(tx) => { txs_map.insert(txid, tx); @@ -538,6 +559,44 @@ impl Mempool { Ok(txos) } + /// Remove the transactions whose prevouts cannot be found in the batch, the mempool or the + /// chain, along with their in-batch descendants, returning the removed txids. + fn drop_unresolvable(&self, txs_map: &mut HashMap) -> Vec { + let mut dropped: HashSet = txs_map + .iter() + .filter(|(_, tx)| { + let external: BTreeSet = get_prev_outpoints(std::iter::once(*tx)) + .into_iter() + .filter(|prevout| match txs_map.get(&prevout.txid) { + Some(parent) => parent.output.get(prevout.vout as usize).is_none(), + None => true, + }) + .collect(); + self.lookup_txos(external).is_err() + }) + .map(|(txid, _)| *txid) + .collect(); + + let mut frontier = dropped.clone(); + while !frontier.is_empty() { + frontier = txs_map + .iter() + .filter(|(txid, tx)| { + !dropped.contains(*txid) + && tx + .input + .iter() + .any(|txin| frontier.contains(&txin.previous_output.txid)) + }) + .map(|(txid, _)| *txid) + .collect(); + dropped.extend(frontier.iter().copied()); + } + + txs_map.retain(|txid, _| !dropped.contains(txid)); + dropped.into_iter().collect() + } + #[trace] fn remove(&mut self, to_remove: HashSet<&Txid>) { self.delta @@ -732,14 +791,18 @@ impl Mempool { // their history and recent entries. fetched_txs.retain(|txid, _| mempool.txstore.get(txid).is_none()); if !fetched_txs.is_empty() { - if let Err(e) = mempool.add(fetched_txs) { + if let Err(e) = mempool.add(fetched_txs.clone()) { + let unresolvable = mempool.drop_unresolvable(&mut fetched_txs); warn!( - "transient failure adding {} fetched txs, skipping until next cycle: {}", + "failed adding {} fetched txs, skipping {} with unresolvable prevouts until next cycle: {}", fetched_txs_len, + unresolvable.len(), e.display_chain() ); mempool.sync_failed_to_index.inc(); - return Ok(MempoolSyncStatus::FailedToIndex); + if unresolvable.is_empty() || mempool.add(fetched_txs).is_err() { + return Ok(MempoolSyncStatus::FailedToIndex); + } } } diff --git a/src/new_index/precache.rs b/src/new_index/precache.rs index 86bd6a9bc..9154ed7e3 100644 --- a/src/new_index/precache.rs +++ b/src/new_index/precache.rs @@ -2,11 +2,12 @@ use crate::chain::address::Address; use crate::errors::*; use crate::new_index::ChainQuery; use crate::util::FullHash; +use error_chain::ChainedError; use rayon::prelude::*; use bitcoin::hashes::{sha256, Hash}; -use bitcoin::hex::FromHex; +use bitcoin::hex::{DisplayHex, FromHex}; use std::fs::File; use std::io; use std::io::prelude::*; @@ -32,7 +33,21 @@ pub fn precache(chain: &ChainQuery, scripthashes: Vec) { if i % 5 == 0 { info!("running pre-cache for scripthash {}/{}", i + 1, total); } - let _ = chain.stats(&scripthash[..]); + // Each TooBigHistory saves a checkpoint further along, so retrying finishes the scan. + loop { + match chain.stats(&scripthash[..]) { + Err(Error(ErrorKind::TooBigHistory, _)) => continue, + Err(e) => { + warn!( + "pre-cache failed scripthash='{}' e='{}'", + scripthash.to_lower_hex_string(), + e.display_chain() + ); + break; + } + Ok(_) => break, + } + } //chain.utxo(&scripthash[..]); }) }); diff --git a/src/new_index/query.rs b/src/new_index/query.rs index 937da8054..c486d43ee 100644 --- a/src/new_index/query.rs +++ b/src/new_index/query.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeSet, HashMap}; use std::sync::{Arc, Mutex, RwLock, RwLockReadGuard, TryLockError}; use std::time::{Duration, Instant}; -use crate::chain::{Network, OutPoint, Transaction, Txid}; +use crate::chain::{deserialize, Network, OutPoint, Transaction, Txid}; use crate::config::Config; use crate::daemon::{Daemon, SubmitPackageResult}; use crate::errors::*; @@ -10,7 +10,9 @@ use crate::new_index::block_template::BlockTemplateCache; use crate::new_index::{ChainQuery, Mempool, ScriptStats, SpendingInput, Utxo}; use crate::util::{is_spendable, BlockId, Bytes, TransactionStatus}; +use bitcoin::hex::FromHex; use electrs_macros::trace; +use error_chain::ChainedError; use hyper::body::Bytes as BodyBytes; #[cfg(feature = "liquid")] @@ -80,10 +82,29 @@ impl Query { #[trace] pub fn broadcast_raw(&self, txhex: &str) -> Result { let txid = self.daemon.broadcast_raw(txhex)?; - let _ = Mempool::ensure_tx(&self.mempool, &self.daemon, txid); + self.index_broadcast(&[txid], &[txhex]); Ok(txid) } + /// Index accepted transactions locally, using the submitted hex rather than a daemon fetch + /// where possible. The daemon already accepted them, so failures are logged, not returned. + fn index_broadcast>(&self, accepted_txids: &[Txid], txhexes: &[S]) { + let known: HashMap = txhexes + .iter() + .filter_map(|txhex| Vec::::from_hex(txhex.as_ref()).ok()) + .filter_map(|bytes| deserialize::(&bytes).ok()) + .map(|tx| (tx.compute_txid(), tx)) + .filter(|(txid, _)| accepted_txids.contains(txid)) + .collect(); + if let Err(e) = Mempool::ensure_txs_with(&self.mempool, &self.daemon, accepted_txids, known) + { + warn!( + "accepted transactions not yet indexed locally, visible after the next mempool sync: {}", + e.display_chain() + ); + } + } + #[trace] pub fn submit_package( &self, @@ -91,12 +112,14 @@ impl Query { maxfeerate: Option, maxburnamount: Option, ) -> Result { - let result = self.daemon.submit_package(txhex, maxfeerate, maxburnamount)?; + let result = self + .daemon + .submit_package(txhex.clone(), maxfeerate, maxburnamount)?; // Add accepted txs to the local mempool so subscription status updates reflect them // immediately (they read from the local mempool), mirroring broadcast_raw() above. let accepted_txids = result.accepted_txids(); if !accepted_txids.is_empty() { - let _ = Mempool::ensure_txs(&self.mempool, &self.daemon, &accepted_txids); + self.index_broadcast(&accepted_txids, &txhex); } Ok(result) } diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index 7bb52f61f..3e2b0887a 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -195,6 +195,12 @@ impl Store { type UtxoMap = HashMap; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum UtxoScanStop { + ScanLimit, + CheckpointLimit, +} + #[derive(Debug)] pub struct Utxo { pub txid: Txid, @@ -293,6 +299,7 @@ pub struct ChainQuery { duration: HistogramVec, network: Network, history_scan_limit: usize, + utxos_checkpoint_limit: usize, checkpoint_merkle_cache: CheckpointMerkleCache, } @@ -637,6 +644,7 @@ impl ChainQuery { light_mode: config.light_mode, network: config.network_type, history_scan_limit: config.history_scan_limit, + utxos_checkpoint_limit: config.utxos_checkpoint_limit, duration: metrics.histogram_vec( HistogramOpts::new("query_duration", "Index query duration (in seconds)"), &["name"], @@ -867,14 +875,20 @@ impl ChainQuery { .map(|row| (row.get_txid(), row.key.confirmed_height as usize)) // XXX: unique() requires keeping an in-memory list of all txids, can we avoid that? .unique() - .skip_while(|(txid, _)| { - // skip until we reach the last_seen_txid - last_seen_txid.map_or(false, |last_seen_txid| last_seen_txid != txid) - }) - .skip(match last_seen_txid { - Some(_) => 1, // skip the last_seen_txid itself - None => 0, + // skip up to and including last_seen_txid, stopping once the scan passes its height + .scan(last_seen_txid.is_none(), |cursor_seen, (txid, height)| { + if *cursor_seen { + Some(Some((txid, height))) + } else if last_seen_txid == Some(&txid) { + *cursor_seen = true; + Some(None) + } else if seek_height.map_or(false, |seek_height| height < seek_height) { + None + } else { + Some(None) + } }) + .flatten() // skip over entries that point to non-existing heights (may happen while new/reorged blocks are being processed) .filter_map(|(txid, height)| Some((txid, headers.header_by_height(height)?))) .take(limit); @@ -923,10 +937,13 @@ impl ChainQuery { // get the last known utxo set and the blockhash it was updated for. // invalidates the cache if the block was orphaned. - let cache: Option<(UtxoMap, usize)> = self - .store - .cache_db - .get(&UtxoCacheRow::key(scripthash)) + let cached_bytes = self.store.cache_db.get(&UtxoCacheRow::key(scripthash)); + if let Some(ref bytes) = cached_bytes { + if UtxoCacheRow::utxo_count(bytes) > self.utxos_checkpoint_limit { + bail!(ErrorKind::TooBigHistory) + } + } + let cache: Option<(UtxoMap, usize)> = cached_bytes .map(|c| bincode::deserialize_little(&c).unwrap()) .and_then(|(utxos_cache, blockhash)| { self.height_by_hash(&blockhash) @@ -936,11 +953,16 @@ impl ChainQuery { let had_cache = cache.is_some(); // update utxo set with new transactions since - let (newutxos, lastblock, processed_items, hit_scan_limit) = cache.map_or_else( + let (newutxos, lastblock, processed_items, stop) = cache.map_or_else( || self.utxo_delta(scripthash, HashMap::new(), 0), |(oldutxos, blockheight)| self.utxo_delta(scripthash, oldutxos, blockheight + 1), ); + if stop == Some(UtxoScanStop::CheckpointLimit) { + bail!(ErrorKind::TooBigHistory) + } + let hit_scan_limit = stop == Some(UtxoScanStop::ScanLimit); + // save updated utxo set to cache, even if we bail out below if let Some(lastblock) = lastblock { if had_cache || hit_scan_limit || processed_items > MIN_HISTORY_ITEMS_TO_CACHE { @@ -991,7 +1013,7 @@ impl ChainQuery { scripthash: &[u8], init_utxos: UtxoMap, start_height: usize, - ) -> (UtxoMap, Option, usize, bool) { + ) -> (UtxoMap, Option, usize, Option) { let _timer = self.start_timer("utxo_delta"); let headers = self.store.indexed_headers.read().unwrap(); let history_iter = self @@ -1007,13 +1029,13 @@ impl ChainQuery { let mut processed_items = 0; let mut lastblock = None; let mut safe_lastblock = None; - let mut hit_scan_limit = false; + let mut stop = None; for (history, blockid) in history_iter { if lastblock.is_some_and(|hash| hash != blockid.hash) { safe_lastblock = lastblock; if processed_items >= self.history_scan_limit { - hit_scan_limit = true; + stop = Some(UtxoScanStop::ScanLimit); break; } } @@ -1031,13 +1053,17 @@ impl ChainQuery { | TxHistoryInfo::Pegin(_) | TxHistoryInfo::Pegout(_) => unreachable!(), }; + + if utxos.len() > self.utxos_checkpoint_limit { + return (utxos, None, processed_items, Some(UtxoScanStop::CheckpointLimit)); + } } - if !hit_scan_limit { + if stop.is_none() { safe_lastblock = lastblock; } - (utxos, safe_lastblock, processed_items, hit_scan_limit) + (utxos, safe_lastblock, processed_items, stop) } pub fn stats(&self, scripthash: &[u8]) -> Result { @@ -2115,6 +2141,15 @@ impl UtxoCacheRow { [b"U", scripthash].concat() } + /// Number of utxos in a serialized row, read from the map's length prefix without + /// deserializing the entries. + fn utxo_count(value: &[u8]) -> usize { + value + .get(..8) + .and_then(|prefix| bincode::deserialize_little::(prefix).ok()) + .map_or(0, |count| std::convert::TryFrom::try_from(count).unwrap_or(usize::MAX)) + } + fn into_row(self) -> DBRow { DBRow { key: bincode::serialize_little(&self.key).unwrap(), @@ -2239,6 +2274,22 @@ mod tests { Ok(()) } + #[cfg(not(feature = "liquid"))] + #[test] + fn utxo_cache_row_count_reads_the_serialized_length() { + let blockid = BlockId { + height: 7, + hash: BlockHash::all_zeros(), + time: 0, + }; + let utxos: UtxoMap = (0..3u32) + .map(|vout| (OutPoint::new(Txid::all_zeros(), vout), (blockid, 1_000))) + .collect(); + let row = UtxoCacheRow::new(&[0u8; 32], &utxos, &BlockHash::all_zeros()); + assert_eq!(UtxoCacheRow::utxo_count(&row.value), 3); + assert_eq!(UtxoCacheRow::utxo_count(&[]), 0); + } + #[test] fn test_completed_prefix_stops_at_first_gap() { let items = [true, true, false, true]; diff --git a/src/rest.rs b/src/rest.rs index e8108ca09..650b68abc 100644 --- a/src/rest.rs +++ b/src/rest.rs @@ -1677,9 +1677,7 @@ impl HttpError { let used_by_confirmed_tx = confirmed_prevouts .iter() .any(|outpoint| outpoint.to_string() == *missing); - let confirmed_parent = missing - .rsplit_once(':') - .and_then(|(txid, _)| Txid::from_str(txid).ok()) + let confirmed_parent = missing_txo_txid(missing) .map(|txid| query.chain().try_tx_confirming_block(&txid)); let confirmed_parent = match confirmed_parent { Some(Ok(blockid)) => blockid.is_some(), @@ -1695,6 +1693,13 @@ impl HttpError { } } +/// Parse the txid out of a `MissingTxo` outpoint string. Elements prefixes its outpoint +/// display with `[elements]`. +fn missing_txo_txid(missing: &str) -> Option { + let (txid, _) = missing.rsplit_once(':')?; + Txid::from_str(txid.strip_prefix("[elements]").unwrap_or(txid)).ok() +} + impl From for HttpError { fn from(msg: String) -> Self { HttpError(StatusCode::BAD_REQUEST, msg) diff --git a/src/rest/mempool_tests.rs b/src/rest/mempool_tests.rs index 976c77591..b6d3eb6c9 100644 --- a/src/rest/mempool_tests.rs +++ b/src/rest/mempool_tests.rs @@ -222,7 +222,9 @@ fn missing_confirmed_parent_is_unavailable() -> Result<()> { let parent = tester.send(&address, "1 BTC".parse().unwrap())?; tester.mine()?; - let missing = errors::Error::from(errors::ErrorKind::MissingTxo(format!("{}:0", parent))); + let missing = errors::Error::from(errors::ErrorKind::MissingTxo( + OutPoint::new(parent, 0).to_string(), + )); let status = HttpError::mempool_prevout(missing, tester.query(), &BTreeSet::new()).0; assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); Ok(()) diff --git a/src/util/electrum_merkle.rs b/src/util/electrum_merkle.rs index 534958585..bce56dc43 100644 --- a/src/util/electrum_merkle.rs +++ b/src/util/electrum_merkle.rs @@ -153,16 +153,24 @@ pub struct CheckpointMerkleCache { inflight_rebuilds: AtomicUsize, inflight_build_bytes: Arc, capacity_bytes: usize, + cache_capacity_bytes: usize, } impl CheckpointMerkleCache { + /// Cached trees may use half of `capacity_bytes`, so a full cache always leaves at least + /// the other half for rebuild working memory. pub fn new(capacity_bytes: usize) -> Self { + Self::with_cache_capacity(capacity_bytes, capacity_bytes / 2) + } + + fn with_cache_capacity(capacity_bytes: usize, cache_capacity_bytes: usize) -> Self { Self { entries: Mutex::new(Vec::new()), build_locks: Mutex::new(HashMap::new()), inflight_rebuilds: AtomicUsize::new(0), inflight_build_bytes: Arc::new(AtomicUsize::new(0)), capacity_bytes, + cache_capacity_bytes: cache_capacity_bytes.min(capacity_bytes), } } @@ -174,7 +182,7 @@ impl CheckpointMerkleCache { } fn insert(&self, cp_height: usize, best_height: usize, cached: Arc) { - if cached.size_bytes > self.capacity_bytes { + if cached.size_bytes > self.cache_capacity_bytes { return; } @@ -194,7 +202,7 @@ impl CheckpointMerkleCache { eviction_positions.sort_unstable_by_key(|pos| entries[*pos].0); let mut evictions_needed = 0; - while total_bytes + cached.size_bytes > self.capacity_bytes + while total_bytes + cached.size_bytes > self.cache_capacity_bytes || remaining_entries >= MAX_CACHE_ENTRIES { let Some(pos) = eviction_positions.get(evictions_needed).copied() else { @@ -517,6 +525,10 @@ mod tests { const GENEROUS_TEST_CACHE_BYTES: usize = 1024 * 1024; + fn full_share_cache(capacity_bytes: usize) -> CheckpointMerkleCache { + CheckpointMerkleCache::with_cache_capacity(capacity_bytes, capacity_bytes) + } + fn leaves(n: usize) -> Vec { (0..n).map(|i| Sha256dHash::hash(&[i as u8])).collect() } @@ -568,7 +580,7 @@ mod tests { let cp_height = 8; let cp_hash = BlockHash::hash(&[7u8]); - let cache = CheckpointMerkleCache::new(GENEROUS_TEST_CACHE_BYTES); + let cache = full_share_cache(GENEROUS_TEST_CACHE_BYTES); let build_count = AtomicUsize::new(0); for height in [0usize, 3, 8] { @@ -607,7 +619,7 @@ mod tests { let old_hash = BlockHash::hash(&[1u8]); let new_hash = BlockHash::hash(&[2u8]); - let cache = CheckpointMerkleCache::new(GENEROUS_TEST_CACHE_BYTES); + let cache = full_share_cache(GENEROUS_TEST_CACHE_BYTES); let build_count = AtomicUsize::new(0); cache @@ -667,7 +679,7 @@ mod tests { let old_hash = BlockHash::hash(&[1u8]); let new_hash = BlockHash::hash(&[2u8]); - let cache = CheckpointMerkleCache::new(GENEROUS_TEST_CACHE_BYTES); + let cache = full_share_cache(GENEROUS_TEST_CACHE_BYTES); let (old_cached, old_ret_sibling) = cache .get_or_build( @@ -723,7 +735,7 @@ mod tests { let entry_leaves = leaves(2); let entry_bytes = build_levels_above_leaves(&entry_leaves).unwrap()[0].len() * std::mem::size_of::(); - let cache = CheckpointMerkleCache::new(5 * entry_bytes); + let cache = full_share_cache(5 * entry_bytes); let leaf = entry_leaves[0]; for cp_height in [10usize, 20, 30, 40, 50] { @@ -772,7 +784,7 @@ mod tests { let entry_leaves = leaves(2); let entry_bytes = build_levels_above_leaves(&entry_leaves).unwrap()[0].len() * std::mem::size_of::(); - let cache = CheckpointMerkleCache::new(5 * entry_bytes); + let cache = full_share_cache(5 * entry_bytes); let leaf = entry_leaves[0]; for cp_height in [10usize, 20, 30, 40, 50] { @@ -819,7 +831,7 @@ mod tests { #[test] fn failed_insert_does_not_partially_evict_lower_entries() { let hash_bytes = std::mem::size_of::(); - let cache = CheckpointMerkleCache::new(5 * hash_bytes); + let cache = full_share_cache(5 * hash_bytes); let make_cached = |height: usize, hashes: usize| { Arc::new(CachedLevels::new( BlockHash::hash(&[height as u8]), @@ -846,7 +858,7 @@ mod tests { let entry_leaves = leaves(2); let entry_bytes = build_levels_above_leaves(&entry_leaves).unwrap()[0].len() * std::mem::size_of::(); - let cache = CheckpointMerkleCache::new((MAX_CACHE_ENTRIES + 100) * entry_bytes); + let cache = full_share_cache((MAX_CACHE_ENTRIES + 100) * entry_bytes); let leaf = entry_leaves[0]; let hash = BlockHash::hash(&[7u8]); @@ -876,7 +888,7 @@ mod tests { let entry_bytes = build_levels_above_leaves(&entry_leaves).unwrap()[0].len() * std::mem::size_of::(); let leaf = entry_leaves[0]; - let cache = CheckpointMerkleCache::new(entry_bytes); + let cache = full_share_cache(entry_bytes); let old_height = 100; let old_hash = BlockHash::hash(&[1u8]); @@ -943,7 +955,7 @@ mod tests { let entry_leaves = leaves(2); let levels = build_levels_above_leaves(&entry_leaves).unwrap(); let entry_bytes = levels[0].len() * std::mem::size_of::(); - let cache = CheckpointMerkleCache::new(2 * entry_bytes); + let cache = full_share_cache(2 * entry_bytes); let leaf = entry_leaves[0]; let new_tip_height = 101; @@ -993,7 +1005,7 @@ mod tests { let cp_hash = BlockHash::hash(&[9u8]); let sibling = sibling_leaf(&leaves, cp_height, 0); - let cache = Arc::new(CheckpointMerkleCache::new(GENEROUS_TEST_CACHE_BYTES)); + let cache = Arc::new(full_share_cache(GENEROUS_TEST_CACHE_BYTES)); let build_count = Arc::new(AtomicUsize::new(0)); let barrier = Arc::new(Barrier::new(8)); @@ -1044,7 +1056,7 @@ mod tests { let cp_hash = BlockHash::hash(&[9u8]); let sibling = sibling_leaf(&leaves, cp_height, 0); - let cache = Arc::new(CheckpointMerkleCache::new(0)); + let cache = Arc::new(full_share_cache(0)); let build_count = Arc::new(AtomicUsize::new(0)); let barrier = Arc::new(Barrier::new(8)); let _keep_build_slot_alive = cache.build_lock_for(cp_height); @@ -1097,7 +1109,7 @@ mod tests { let cp_hash = BlockHash::hash(&[9u8]); let sibling = sibling_leaf(&leaves, cp_height, 0); - let cache = CheckpointMerkleCache::new(0); + let cache = full_share_cache(0); let slot_a = cache.build_lock_for(cp_height); let slot_b = cache.build_lock_for(cp_height); @@ -1138,7 +1150,7 @@ mod tests { let leaf = entry_leaves[0]; let blocked_height = 1_000usize; - let cache = CheckpointMerkleCache::new(estimated_peak_build_bytes(blocked_height)); + let cache = full_share_cache(estimated_peak_build_bytes(blocked_height)); let uncached_height = 100; let uncached_hash = BlockHash::hash(&[1u8]); @@ -1204,7 +1216,7 @@ mod tests { let cp_hash = BlockHash::hash(&[1u8]); let sibling = sibling_leaf(&leaves, cp_height, 0); - let cache = Arc::new(CheckpointMerkleCache::new(GENEROUS_TEST_CACHE_BYTES)); + let cache = Arc::new(full_share_cache(GENEROUS_TEST_CACHE_BYTES)); let _keep_slot_alive = cache.build_lock_for(cp_height); let panicking = { @@ -1263,7 +1275,7 @@ mod tests { let sibling = sibling_leaf(&leaves, cp_height, 0); let cp_hash = BlockHash::hash(&[3u8]); - let cache = CheckpointMerkleCache::new(GENEROUS_TEST_CACHE_BYTES); + let cache = full_share_cache(GENEROUS_TEST_CACHE_BYTES); cache .get_or_build( cp_height, @@ -1292,7 +1304,7 @@ mod tests { let sibling = sibling_leaf(&leaves, cp_height, 0); let cp_hash = BlockHash::hash(&[3u8]); - let cache = Arc::new(CheckpointMerkleCache::new(GENEROUS_TEST_CACHE_BYTES)); + let cache = Arc::new(full_share_cache(GENEROUS_TEST_CACHE_BYTES)); cache .get_or_build( cp_height, @@ -1360,7 +1372,7 @@ mod tests { .map(|h| estimated_peak_build_bytes(*h)) .sum(); - let cache = Arc::new(CheckpointMerkleCache::new(capacity_bytes)); + let cache = Arc::new(full_share_cache(capacity_bytes)); let inside_build = Arc::new(Barrier::new(HOLDERS + 1)); let release_build = Arc::new(Barrier::new(HOLDERS + 1)); @@ -1426,7 +1438,7 @@ mod tests { + estimated_peak_build_bytes(extra_height) - 1; - let cache = Arc::new(CheckpointMerkleCache::new(capacity_bytes)); + let cache = Arc::new(full_share_cache(capacity_bytes)); cache .get_or_build( resident_height, @@ -1486,4 +1498,78 @@ mod tests { release_build.wait(); holder.join().unwrap(); } + + #[test] + fn a_full_cache_still_admits_concurrent_rebuilds() { + let build_height = 1_000usize; + let capacity_bytes = 4 * estimated_peak_build_bytes(build_height); + let cache = Arc::new(CheckpointMerkleCache::new(capacity_bytes)); + + let filler_leaves = leaves(1 << 12); + let filler_bytes = CachedLevels::new( + BlockHash::hash(&[0u8]), + build_levels_above_leaves(&filler_leaves).unwrap(), + ) + .size_bytes; + let mut height = 1usize << 20; + let mut resident = 0usize; + while resident + filler_bytes <= capacity_bytes / 2 { + let hash = BlockHash::hash(&height.to_le_bytes()); + cache + .get_or_build( + height, + || usize::MAX, + 1, + || Ok((hash, filler_leaves[0])), + || Ok((hash, build_levels_above_leaves(&filler_leaves)?, filler_leaves[0])), + ) + .unwrap(); + resident += filler_bytes; + height += 1; + } + assert!(resident > 0); + + let entered = Arc::new(AtomicUsize::new(0)); + let release = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let handles: Vec<_> = vec![build_height, build_height + 1] + .into_iter() + .map(|miss_height: usize| { + let cache = cache.clone(); + let entered = entered.clone(); + let release = release.clone(); + thread::spawn(move || { + let hash = BlockHash::hash(&[miss_height as u8]); + let leaf = Sha256dHash::hash(&[0u8]); + cache.get_or_build( + miss_height, + || usize::MAX, + 100, + || Ok((hash, leaf)), + || { + entered.fetch_add(1, Ordering::SeqCst); + while !release.load(Ordering::SeqCst) { + thread::sleep(std::time::Duration::from_millis(1)); + } + Ok((hash, build_levels_above_leaves(&[leaf])?, leaf)) + }, + ) + }) + }) + .collect(); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while entered.load(Ordering::SeqCst) < 2 + && !handles.iter().any(|handle| handle.is_finished()) + && std::time::Instant::now() < deadline + { + thread::sleep(std::time::Duration::from_millis(1)); + } + release.store(true, Ordering::SeqCst); + for handle in handles { + assert!( + handle.join().unwrap().is_ok(), + "cached trees must not leave room for only a single rebuild" + ); + } + } } diff --git a/tests/common.rs b/tests/common.rs index ee985d077..6ebd81000 100644 --- a/tests/common.rs +++ b/tests/common.rs @@ -128,6 +128,7 @@ impl TestRunner { precache_scripts: None, utxos_limit: 100, history_scan_limit: 100_000, + utxos_checkpoint_limit: 1_000, electrum_txs_limit: 100, electrum_subscription_limit: 10_000, electrum_checkpoint_proof_concurrency_limit: 2, diff --git a/tests/config.rs b/tests/config.rs index 862cee1c2..2962177c2 100644 --- a/tests/config.rs +++ b/tests/config.rs @@ -112,3 +112,71 @@ fn checkpoint_proof_concurrency_limit_can_be_overridden() { stderr ); } + +fn assert_rejected(extra_args: &[&str], needle: &str) { + let temp_dir = tempfile::tempdir().unwrap(); + let output = run_electrs(temp_dir.path(), extra_args); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!( + !output.status.success(), + "electrs accepted {:?}", + extra_args + ); + assert!( + stderr.contains(needle), + "expected '{}' in stderr, got: {}", + needle, + stderr + ); +} + +#[test] +fn zero_history_scan_limit_is_rejected() { + assert_rejected( + &["--history-scan-limit", "0"], + "--history-scan-limit must be at least 1", + ); +} + +#[test] +fn utxos_checkpoint_limit_below_utxos_limit_is_rejected() { + assert_rejected( + &["--utxos-limit", "500", "--utxos-checkpoint-limit", "499"], + "--utxos-checkpoint-limit (499) must be >= --utxos-limit (500)", + ); +} + +#[test] +fn disabled_write_timeout_requires_an_unlimited_budget_or_max_age() { + assert_rejected( + &["--electrum-rpc-write-timeout", "0"], + "requires a non-zero --electrum-rpc-write-timeout or --electrum-rpc-conn-max-age", + ); + + for extra_args in [ + [ + "--electrum-rpc-write-timeout", + "0", + "--electrum-rpc-global-response-budget-bytes", + "0", + ], + [ + "--electrum-rpc-write-timeout", + "0", + "--electrum-rpc-conn-max-age", + "600", + ], + ] { + let temp_dir = tempfile::tempdir().unwrap(); + let mut args = vec!["-vv"]; + args.extend(extra_args); + let output = run_electrs(temp_dir.path(), &args); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!( + stderr.contains("electrum_rpc_write_timeout: None"), + "{:?} should pass validation, stderr: {}", + extra_args, + stderr + ); + } +} diff --git a/tests/scan_limits.rs b/tests/scan_limits.rs index 692849826..8fe65446c 100644 --- a/tests/scan_limits.rs +++ b/tests/scan_limits.rs @@ -305,3 +305,90 @@ fn test_history_cursor_unmatched_returns_empty() -> Result<()> { rest_handle.stop(); Ok(()) } + +#[cfg(not(feature = "liquid"))] +#[test] +fn test_history_cursor_from_another_script_does_not_trip_the_scan_limit() -> Result<()> { + let (rest_handle, rest_addr, mut tester) = + common::init_rest_tester_with(|c| c.history_scan_limit = 5).unwrap(); + + let addr1 = tester.newaddress()?; + let addr2 = tester.newaddress()?; + let amount = bitcoin::Amount::from_sat(1_000); + + for _ in 0..4 { + tester.send_multi(&addr1, amount, 3)?; + tester.mine()?; + } + // Confirmed at a height where addr1 has no history rows. + let foreign_txid = tester.send(&addr2, amount)?; + tester.mine()?; + + let res = get_json( + rest_addr, + &format!("/address/{}/txs/chain/{}", addr1, foreign_txid), + )?; + assert_eq!(res.as_array().expect("array of txs").len(), 0); + + rest_handle.stop(); + Ok(()) +} + +#[cfg(not(feature = "liquid"))] +#[test] +fn test_precache_retries_until_the_scan_completes() -> Result<()> { + use electrs::new_index::precache; + + let (rest_handle, rest_addr, mut tester) = + common::init_rest_tester_with(|c| c.history_scan_limit = 5).unwrap(); + + let addr1 = tester.newaddress()?; + let amount = bitcoin::Amount::from_sat(1_000); + for _ in 0..12 { + tester.send(&addr1, amount)?; + tester.mine()?; + } + + let scripthash = precache::compute_script_hash(addr1.script_pubkey().as_bytes()); + precache::precache(tester.query().chain(), vec![scripthash]); + + let stats = get_json(rest_addr, &format!("/address/{}", addr1))?; + assert_eq!(stats["chain_stats"]["funded_txo_count"].as_u64(), Some(12)); + + rest_handle.stop(); + Ok(()) +} + +#[cfg(not(feature = "liquid"))] +#[test] +fn test_utxo_checkpoint_limit_bounds_the_saved_set() -> Result<()> { + let (rest_handle, rest_addr, mut tester) = common::init_rest_tester_with(|c| { + c.utxos_limit = 10; + c.utxos_checkpoint_limit = 20; + c.history_scan_limit = 128; + }) + .unwrap(); + + let addr1 = tester.newaddress()?; + let amount = bitcoin::Amount::from_sat(1_000); + for _ in 0..4 { + tester.send_multi(&addr1, amount, 64)?; + tester.mine()?; + } + + for _ in 0..4 { + let resp = get_allow_error(rest_addr, &format!("/address/{}/utxo", addr1))?; + assert_eq!(resp.status(), 400); + } + + let scripthash = + electrs::new_index::precache::compute_script_hash(addr1.script_pubkey().as_bytes()); + let key = [b"U".as_slice(), &scripthash[..]].concat(); + assert!( + tester.query().chain().store().cache_db().get(&key).is_none(), + "a utxo set above --utxos-checkpoint-limit must not be checkpointed" + ); + + rest_handle.stop(); + Ok(()) +} From b33362f7d3cdfa791983b7e491b358b0ddfca319 Mon Sep 17 00:00:00 2001 From: Oleksandr Fedorenko Date: Wed, 30 Sep 2026 18:46:38 +0300 Subject: [PATCH 25/28] test: make Electrum response budget recovery check portable --- tests/common.rs | 10 ++++++++++ tests/electrum_response_limits.rs | 15 ++++++--------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/tests/common.rs b/tests/common.rs index 6ebd81000..3ca849501 100644 --- a/tests/common.rs +++ b/tests/common.rs @@ -475,6 +475,16 @@ impl TestRunner { .map(|metric| metric.get_counter().value()) .sum() } + + pub fn counter_value(&self, name: &str) -> f64 { + self.metrics + .gather() + .iter() + .filter(|family| family.name() == name) + .flat_map(|family| family.get_metric()) + .map(|metric| metric.get_counter().value()) + .sum() + } } pub fn init_rest_tester() -> Result<(rest::Handle, net::SocketAddr, TestRunner)> { diff --git a/tests/electrum_response_limits.rs b/tests/electrum_response_limits.rs index 039ba310d..2585e3b93 100644 --- a/tests/electrum_response_limits.rs +++ b/tests/electrum_response_limits.rs @@ -135,7 +135,7 @@ fn response_line_limits_preserve_ids_and_skip_later_commands() { fn response_budget_recovers_when_stalled_batch_writers_time_out() { let cap = 16 * 1024 * 1024; let banner_bytes = 4 * 1024 * 1024 - 4096; - let (_server, addr, _tester) = common::init_electrum_tester_with_config(|config| { + let (_server, addr, tester) = common::init_electrum_tester_with_config(|config| { config.electrum_rpc_max_response_num_bytes = cap; config.electrum_rpc_global_response_budget_bytes = cap; config.electrum_rpc_write_timeout = Some(Duration::from_secs(5)); @@ -146,8 +146,7 @@ fn response_budget_recovers_when_stalled_batch_writers_time_out() { let mut holders = Vec::new(); for _ in 0..2 { - // The 8 MiB reply fills the default TCP buffers. Leave the negotiated - // receive window unchanged so reset delivery is reliable on macOS. + // The 8 MiB reply fills the default TCP buffers. let mut client = Client::connect(addr); client.send(&json!([ {"id":1,"method":"server.banner"}, @@ -182,12 +181,10 @@ fn response_budget_recovers_when_stalled_batch_writers_time_out() { } let (reply, _) = healthy.call(json!({"id":3,"method":"server.banner"})); assert_eq!(reply["result"].as_str().unwrap().len(), banner_bytes); - // The first holder's deadline starts first. Recovery can precede the - // second holder's timeout, so keep that client unread while checking reset. - // Buffered bytes may precede the reset on the first connection. - let error = std::io::copy(&mut holders[0].0, &mut std::io::sink()) - .expect_err("a timed-out writer must reset the connection"); - assert_eq!(error.kind(), std::io::ErrorKind::ConnectionReset); + assert!( + tester.counter_value("electrum_client_write_timeouts_total") >= 1.0, + "budget recovered without a stalled writer timing out" + ); } #[test] From 371ca75e9add5082fa9c50258a4e0a3227485b1c Mon Sep 17 00:00:00 2001 From: Oleksandr Fedorenko Date: Wed, 30 Sep 2026 19:18:10 +0300 Subject: [PATCH 26/28] fix: index the daemon's actual witness after transaction broadcast --- src/chain.rs | 3 +- src/daemon.rs | 42 ++++++++++++++++++++- src/elements/mod.rs | 4 ++ src/new_index/query.rs | 74 ++++++++++++++++++++++++++++++------- tests/rest.rs | 84 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 192 insertions(+), 15 deletions(-) diff --git a/src/chain.rs b/src/chain.rs index 0475223e6..680b45f57 100644 --- a/src/chain.rs +++ b/src/chain.rs @@ -2,7 +2,7 @@ pub use bitcoin::{ address, blockdata::block::Header as BlockHeader, blockdata::script, consensus::deserialize, hash_types::TxMerkleNode, Address, Block, BlockHash, OutPoint, ScriptBuf as Script, Sequence, - Transaction, TxIn, TxOut, Txid, + Transaction, TxIn, TxOut, Txid, Wtxid, }; #[cfg(feature = "liquid")] @@ -11,6 +11,7 @@ pub use { elements::{ address, confidential, encode::deserialize, script, Address, AssetId, Block, BlockHash, BlockHeader, OutPoint, Script, Sequence, Transaction, TxIn, TxMerkleNode, TxOut, Txid, + Wtxid, }, }; diff --git a/src/daemon.rs b/src/daemon.rs index 6feb7f231..ece3e937b 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -23,7 +23,7 @@ use elements::encode::{deserialize, serialize_hex}; use electrs_macros::trace; -use crate::chain::{Block, BlockHash, BlockHeader, Network, Transaction, Txid}; +use crate::chain::{Block, BlockHash, BlockHeader, Network, Transaction, Txid, Wtxid}; use crate::metrics::{CounterVec, HistogramOpts, HistogramVec, MetricOpts, Metrics}; use crate::signal::Waiter; use crate::util::{HeaderList, DEFAULT_BLOCKHASH}; @@ -245,6 +245,19 @@ impl SubmitPackageResult { .collect() } + /// (txid, wtxid) of the accepted transactions that are in the daemon's mempool exactly as + /// submitted. `other-wtxid` results are excluded: a same-txid transaction with a different + /// witness was already there and the submitted one was ignored. + pub fn accepted_as_submitted(&self) -> Vec<(Txid, Wtxid)> { + self.tx_results + .iter() + .filter(|(_, tx)| tx.error.is_none() && tx.other_wtxid.is_none()) + .filter_map(|(wtxid, tx)| { + Some((Txid::from_str(&tx.txid).ok()?, Wtxid::from_str(wtxid).ok()?)) + }) + .collect() + } + /// Build the response for the Electrum `blockchain.transaction.broadcast_package` method. /// /// When `verbose` is true, the full `submitpackage` result is returned. Otherwise a compact @@ -1641,6 +1654,33 @@ mod tests { Duration::from_millis(n) } + #[test] + fn accepted_as_submitted_skips_errors_and_other_witnesses() { + use super::SubmitPackageResult; + use crate::chain::{Txid, Wtxid}; + use std::str::FromStr; + + let id = |n: u8| format!("{:02x}", n).repeat(32); + let result: SubmitPackageResult = serde_json::from_value(json!({ + "package_msg": "success", + "tx-results": { + id(1): { "txid": id(11), "vsize": 100, "fees": { "base": 0.0001 } }, + id(2): { "txid": id(12), "other-wtxid": id(3) }, + id(4): { "txid": id(14), "error": "bad-txns-inputs-missingorspent" }, + }, + })) + .unwrap(); + + assert_eq!( + result.accepted_as_submitted(), + vec![( + Txid::from_str(&id(11)).unwrap(), + Wtxid::from_str(&id(1)).unwrap() + )] + ); + assert_eq!(result.accepted_txids().len(), 2); + } + #[test] fn reconnect_failures_are_backed_off_before_retrying() { let mut attempts = 0; diff --git a/src/elements/mod.rs b/src/elements/mod.rs index e0d044c2b..957dc50d0 100644 --- a/src/elements/mod.rs +++ b/src/elements/mod.rs @@ -97,10 +97,14 @@ pub mod ebcompact { pub trait TxidCompat { fn compute_txid(&self) -> elements::Txid; + fn compute_wtxid(&self) -> elements::Wtxid; } impl TxidCompat for elements::Transaction { fn compute_txid(&self) -> elements::Txid { self.txid() } + fn compute_wtxid(&self) -> elements::Wtxid { + self.wtxid() + } } } diff --git a/src/new_index/query.rs b/src/new_index/query.rs index c486d43ee..70a6bb7d9 100644 --- a/src/new_index/query.rs +++ b/src/new_index/query.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeSet, HashMap}; use std::sync::{Arc, Mutex, RwLock, RwLockReadGuard, TryLockError}; use std::time::{Duration, Instant}; -use crate::chain::{deserialize, Network, OutPoint, Transaction, Txid}; +use crate::chain::{deserialize, Network, OutPoint, Transaction, Txid, Wtxid}; use crate::config::Config; use crate::daemon::{Daemon, SubmitPackageResult}; use crate::errors::*; @@ -82,20 +82,15 @@ impl Query { #[trace] pub fn broadcast_raw(&self, txhex: &str) -> Result { let txid = self.daemon.broadcast_raw(txhex)?; - self.index_broadcast(&[txid], &[txhex]); + // sendrawtransaction also succeeds when a same-txid tx with a different witness is + // already in the mempool, so the submitted hex may not be the daemon's copy. + self.index_broadcast(&[txid], HashMap::new()); Ok(txid) } - /// Index accepted transactions locally, using the submitted hex rather than a daemon fetch - /// where possible. The daemon already accepted them, so failures are logged, not returned. - fn index_broadcast>(&self, accepted_txids: &[Txid], txhexes: &[S]) { - let known: HashMap = txhexes - .iter() - .filter_map(|txhex| Vec::::from_hex(txhex.as_ref()).ok()) - .filter_map(|bytes| deserialize::(&bytes).ok()) - .map(|tx| (tx.compute_txid(), tx)) - .filter(|(txid, _)| accepted_txids.contains(txid)) - .collect(); + /// Index accepted transactions locally, taking those in `known` as-is and fetching the rest + /// from the daemon. The daemon already accepted them, so failures are logged, not returned. + fn index_broadcast(&self, accepted_txids: &[Txid], known: HashMap) { if let Err(e) = Mempool::ensure_txs_with(&self.mempool, &self.daemon, accepted_txids, known) { warn!( @@ -119,7 +114,8 @@ impl Query { // immediately (they read from the local mempool), mirroring broadcast_raw() above. let accepted_txids = result.accepted_txids(); if !accepted_txids.is_empty() { - self.index_broadcast(&accepted_txids, &txhex); + let known = submitted_txs(&txhex, &result.accepted_as_submitted()); + self.index_broadcast(&accepted_txids, known); } Ok(result) } @@ -369,3 +365,55 @@ impl Query { } } +/// Parse the submitted package transactions that the daemon holds exactly as submitted, matching +/// both txid and wtxid. +fn submitted_txs(txhexes: &[String], submitted: &[(Txid, Wtxid)]) -> HashMap { + txhexes + .iter() + .filter_map(|txhex| Vec::::from_hex(txhex).ok()) + .filter_map(|bytes| deserialize::(&bytes).ok()) + .filter(|tx| submitted.contains(&(tx.compute_txid(), tx.compute_wtxid()))) + .map(|tx| (tx.compute_txid(), tx)) + .collect() +} + +#[cfg(all(test, not(feature = "liquid")))] +mod tests { + use super::submitted_txs; + use bitcoin::consensus::encode::serialize_hex; + use bitcoin::{ + absolute, transaction, OutPoint, ScriptBuf, Sequence, Transaction, TxIn, Witness, + }; + + fn tx_with_witness(item: &[u8]) -> Transaction { + Transaction { + version: transaction::Version::TWO, + lock_time: absolute::LockTime::ZERO, + input: vec![TxIn { + previous_output: OutPoint::null(), + script_sig: ScriptBuf::new(), + sequence: Sequence::MAX, + witness: Witness::from_slice(&[item]), + }], + output: vec![], + } + } + + #[test] + fn submitted_txs_require_matching_txid_and_wtxid() { + let submitted = tx_with_witness(&[1]); + let other = tx_with_witness(&[2]); + let txhexes = vec![serialize_hex(&submitted)]; + + let matching = [(submitted.compute_txid(), submitted.compute_wtxid())]; + assert_eq!( + submitted_txs(&txhexes, &matching).get(&submitted.compute_txid()), + Some(&submitted) + ); + + let other_witness = [(submitted.compute_txid(), other.compute_wtxid())]; + assert!(submitted_txs(&txhexes, &other_witness).is_empty()); + + assert!(submitted_txs(&txhexes, &[]).is_empty()); + } +} diff --git a/tests/rest.rs b/tests/rest.rs index fdffec1b3..67287cff3 100644 --- a/tests/rest.rs +++ b/tests/rest.rs @@ -1367,6 +1367,90 @@ fn test_rest_package_updates_mempool() -> Result<()> { Ok(()) } +/// Build two spends of a fresh `OP_DROP OP_TRUE` P2WSH output that share a txid but differ in +/// witness, returning (txid, first variant hex, second variant hex). +#[cfg(not(feature = "liquid"))] +fn same_txid_witness_variants(tester: &mut common::TestRunner) -> Result<(Txid, String, String)> { + use bitcoin::blockdata::opcodes::all::{OP_DROP, OP_PUSHNUM_1}; + use bitcoin::consensus::encode::serialize_hex; + use bitcoin::{ + absolute, transaction, Address, OutPoint, ScriptBuf, Sequence, Transaction, TxIn, TxOut, + Witness, + }; + + let witness_script = ScriptBuf::builder() + .push_opcode(OP_DROP) + .push_opcode(OP_PUSHNUM_1) + .into_script(); + let p2wsh = Address::p2wsh(&witness_script, bitcoin::Network::Regtest); + let funding_txid = tester.send(&p2wsh, Amount::from_sat(100_000))?; + let funding_tx = tester.get_raw_transaction(funding_txid)?; + tester.mine()?; + let vout = funding_tx + .output + .iter() + .position(|txout| txout.script_pubkey == p2wsh.script_pubkey()) + .expect("funding output") as u32; + + let spend = |witness_item: &[u8]| Transaction { + version: transaction::Version::TWO, + lock_time: absolute::LockTime::ZERO, + input: vec![TxIn { + previous_output: OutPoint::new(funding_txid, vout), + script_sig: ScriptBuf::new(), + sequence: Sequence::ENABLE_RBF_NO_LOCKTIME, + witness: Witness::from_slice(&[witness_item, witness_script.as_bytes()]), + }], + output: vec![TxOut { + value: Amount::from_sat(90_000), + script_pubkey: p2wsh.script_pubkey(), + }], + }; + let first = spend(&[0x01]); + let second = spend(&[0x02; 32]); + assert_eq!(first.compute_txid(), second.compute_txid()); + assert_ne!(first.compute_wtxid(), second.compute_wtxid()); + + Ok(( + first.compute_txid(), + serialize_hex(&first), + serialize_hex(&second), + )) +} + +/// Regression test: when bitcoind already holds a transaction with the same txid but a different +/// witness, broadcasting the other variant succeeds without replacing it. electrs must index the +/// daemon's copy, not the submitted one. +#[cfg(not(feature = "liquid"))] +#[test] +fn test_rest_broadcast_indexes_the_daemons_witness() -> Result<()> { + let (rest_handle, rest_addr, mut tester) = common::init_rest_tester().unwrap(); + + for endpoint in ["/tx", "/txs/package"] { + let (txid, in_mempool_hex, submitted_hex) = same_txid_witness_variants(&mut tester)?; + let _: Value = tester + .node_client() + .call("sendrawtransaction", &[in_mempool_hex.clone().into()])?; + + let resp = match endpoint { + "/tx" => ureq::post(&format!("http://{}/tx", rest_addr)).send(&submitted_hex)?, + _ => ureq::post(&format!("http://{}/txs/package", rest_addr)) + .send_json([&submitted_hex])?, + }; + assert_eq!(resp.status(), 200, "{} rejected the variant", endpoint); + + assert_eq!( + get_plain(rest_addr, &format!("/tx/{}/hex", txid))?, + in_mempool_hex, + "{} indexed the submitted witness instead of the daemon's", + endpoint + ); + } + + rest_handle.stop(); + Ok(()) +} + #[cfg(not(feature = "liquid"))] #[test] fn test_mempool_ensure_txs_concurrency() -> Result<()> { From cb543290826a71cebadf647def2a09a2452bf6f5 Mon Sep 17 00:00:00 2001 From: Oleksandr Fedorenko Date: Thu, 1 Oct 2026 12:19:10 +0300 Subject: [PATCH 27/28] fix(mempool): report partial sync as FailedToIndex --- src/new_index/mempool.rs | 6 +++-- src/new_index/schema.rs | 54 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/src/new_index/mempool.rs b/src/new_index/mempool.rs index 22efa964c..691795d3d 100644 --- a/src/new_index/mempool.rs +++ b/src/new_index/mempool.rs @@ -782,6 +782,7 @@ impl Mempool { // Add fetched transactions to our view of the mempool trace!("indexing {} new mempool transactions", fetched_txs.len()); + let mut status = MempoolSyncStatus::Synced; if !fetched_txs.is_empty() { let fetched_txs_len = fetched_txs.len(); let mut mempool = mempool.write().unwrap(); @@ -803,6 +804,7 @@ impl Mempool { if unresolvable.is_empty() || mempool.add(fetched_txs).is_err() { return Ok(MempoolSyncStatus::FailedToIndex); } + status = MempoolSyncStatus::FailedToIndex; } } @@ -816,9 +818,9 @@ impl Mempool { } } - trace!("mempool is synced"); + trace!("mempool update finished status={:?}", status); - Ok(MempoolSyncStatus::Synced) + Ok(status) } } diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index 3e2b0887a..a49639513 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -2258,6 +2258,60 @@ mod tests { use super::*; use crate::test_common as common; + #[cfg(not(feature = "liquid"))] + #[test] + fn partial_mempool_sync_reports_failure_and_keeps_resolvable_txs() -> common::Result<()> { + use crate::new_index::{Mempool, MempoolSyncStatus}; + use bitcoin::consensus::encode::serialize_hex; + use serde_json::{json, Value}; + + let mut tester = common::TestRunner::new()?; + let address = tester.newaddress()?; + let funding_txid = tester.send_multi(&address, bitcoin::Amount::ONE_BTC, 2)?; + tester.mine()?; + + let spend = |vout: u32| -> common::Result { + let tx = Transaction { + version: bitcoin::transaction::Version::TWO, + lock_time: bitcoin::absolute::LockTime::ZERO, + input: vec![bitcoin::TxIn { + previous_output: OutPoint::new(funding_txid, vout), + ..Default::default() + }], + output: vec![TxOut { + value: bitcoin::Amount::from_sat(99_990_000), + script_pubkey: tester.newaddress()?.script_pubkey(), + }], + }; + let signed: Value = tester + .node_client() + .call("signrawtransactionwithwallet", &[json!(serialize_hex(&tx))])?; + Ok(tester + .node_client() + .call("sendrawtransaction", &[signed["hex"].clone()])?) + }; + let unresolvable_txid = spend(0)?; + let resolvable_txid = spend(1)?; + + tester.query().chain().store.txstore_db.delete_rows( + vec![DBRow { + key: TxOutRow::key(&OutPoint::new(funding_txid, 0)), + value: vec![], + }], + DBFlush::Enable, + ); + + let tip = tester.get_best_block_hash()?; + assert_eq!( + Mempool::update(tester.mempool(), tester.daemon(), &tip)?, + MempoolSyncStatus::FailedToIndex + ); + let mempool = tester.mempool().read().unwrap(); + assert!(mempool.lookup_txn(&resolvable_txid).is_some()); + assert!(mempool.lookup_txn(&unresolvable_txid).is_none()); + Ok(()) + } + #[test] fn corrupt_single_key_rows_return_errors() -> common::Result<()> { let mut tester = common::TestRunner::new()?; From 80c273fd501b6c4f78a88c4d916777a336ce10de Mon Sep 17 00:00:00 2001 From: Oleksandr Fedorenko Date: Thu, 1 Oct 2026 12:24:29 +0300 Subject: [PATCH 28/28] test(scan_limits): fund the foreign tx explicitly so coin selection can't spend addr1 --- tests/scan_limits.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scan_limits.rs b/tests/scan_limits.rs index 8fe65446c..83377118d 100644 --- a/tests/scan_limits.rs +++ b/tests/scan_limits.rs @@ -321,7 +321,7 @@ fn test_history_cursor_from_another_script_does_not_trip_the_scan_limit() -> Res tester.mine()?; } // Confirmed at a height where addr1 has no history rows. - let foreign_txid = tester.send(&addr2, amount)?; + let foreign_txid = tester.send_multi(&addr2, amount, 1)?; tester.mine()?; let res = get_json(