Fix scan limits mempool hardening

README.md

@@ -65,11 +65,19 @@ In addition to electrs's original configuration options, a few new options are a
 - `--http-addr <addr:port>` - HTTP server address/port to listen on (default: `127.0.0.1:3000`).
 - `--lightmode` - enable light mode (see above)
 - `--cors <origins>` - origins allowed to make cross-site request (optional, defaults to none).
-- `--address-search` - enables the by-prefix address search index.
+- `--address-search` - enables the by-prefix address search index. Best-effort: entries are
+  written when a block is indexed but are not removed if that block is later reorged out, so
+  the index may return addresses with no transaction in the current best chain. Run a full
+  reindex to clear stale entries if this matters for your deployment.
 - `--index-unspendables` - enables indexing of provably unspendable outputs.
 - `--enable-mining-rest` - enables cached mining-related HTTP endpoints.
 - `--utxos-limit <num>` - maximum number of utxos to return per address.
+- `--history-scan-limit <num>` - approximate number of history rows scanned per address lookup before failing with a "too popular" error; progress is checkpointed so repeated lookups make headway. Must be at least `1`. Default: `100000`.
+- `--utxos-checkpoint-limit <num>` - maximum number of utxos held in a partially scanned utxo set and its saved checkpoint, and of unconfirmed history entries scanned per address. Must be at least `--utxos-limit`. Default: 10 times `--utxos-limit`.
 - `--electrum-txs-limit <num>` - maximum number of txs to return per address in the electrum server (does not apply for the http api).
+- `--electrum-rpc-write-timeout <seconds>` - maximum time to transmit a complete Electrum reply, including a whole batch and its newline, an error reply, or a subscription notification. The deadline starts at the first write and is not extended by partial progress. Expiry closes the connection and releases its response buffers. Idle subscriptions and command execution are unaffected. Default: `30`, matching the RPC proxy's `CLIENT_WRITE_TIMEOUT_SECS`; `0` disables the bound, which is only accepted with `--electrum-rpc-conn-max-age` set or the global response budget disabled. The `electrum_client_write_timeouts_total` counter records connections closed by this deadline.
+- `--electrum-rpc-max-response-num-bytes <num>` - maximum size, in bytes, of a single Electrum solicited reply line (brackets, commas, error objects, and newline included). Overflowing replies return a correlated error (code 1) and the rest of the batch is not executed; the command that overflowed may already have completed. Subscription notifications are exempt. `0` disables the cap. Default: `8388608` (8 MiB) on Bitcoin, `33554432` (32 MiB) on Liquid (sized for a 2016-header window during a dynafed parameter vote).
+- `--electrum-rpc-global-response-budget-bytes <num>` - aggregate cap, in bytes, on Electrum solicited-reply buffer memory retained across all connections at any instant, including batch replies held while later batch elements execute. Subscription notifications and queued request lines are not counted. Small replies below a 16 KiB per-connection floor are exempt; above the floor each reply charges the budget in 64 KiB chunks (or a smaller final chunk) and releases on drop. Requests that would exceed the budget are rejected with a server error (code 2) instead of allocating, and the connection stays usable. Must be `>= --electrum-rpc-max-response-num-bytes`, and requires a non-zero `--electrum-rpc-write-timeout` or `--electrum-rpc-conn-max-age`. `0` disables the budget. Default: `67108864` (64 MiB) on Bitcoin, `268435456` (256 MiB) on Liquid.
 - `--electrum-banner <text>` - welcome banner text for electrum server.
 
 Additional options with the `liquid` feature:

src/bin/electrs.rs

@@ -8,7 +8,7 @@ extern crate log;
 extern crate electrs;
 
 use crossbeam_channel::{self as channel};
-use error_chain::ChainedError;
+use error_chain::{bail, ChainedError};
 use std::{env, process, thread};
 use std::sync::{Arc, RwLock};
 use std::time::Duration;
@@ -20,7 +20,9 @@ use electrs::{
     electrum::RPC as ElectrumRPC,
     errors::*,
     metrics::Metrics,
-    new_index::{precache, zmq, ChainQuery, FetchFrom, Indexer, Mempool, Query, Store},
+    new_index::{
+        precache, zmq, ChainQuery, FetchFrom, Indexer, Mempool, MempoolSyncStatus, Query, Store,
+    },
     rest,
     signal::Waiter,
 };
@@ -35,6 +37,14 @@ use electrs::metrics::MetricOpts;
 /// Default salt rotation interval in seconds (24 hours)
 const DEFAULT_SALT_ROTATION_INTERVAL_SECS: u64 = 24 * 3600;
 
+/// Maximum number of `FailedToIndex` retries during the initial mempool
+/// sync before giving up.
+const MAX_INITIAL_MEMPOOL_SYNC_RETRIES: u32 = 100;
+
+/// Maximum number of consecutive `FailedToIndex` main loop cycles before giving up,
+/// about an hour at the 5 second loop interval.
+const MAX_CONSECUTIVE_MEMPOOL_SYNC_FAILURES: u32 = 720;
+
 fn fetch_from(config: &Config, store: &Store) -> FetchFrom {
     let mut jsonrpc_import = config.jsonrpc_import;
     if !jsonrpc_import {
@@ -122,18 +132,32 @@ fn run_server(config: Arc<Config>, salt_rwlock: Arc<RwLock<String>>) -> Result<(
         Arc::clone(&config),
     )));
 
-    // A mempool update is retried whenever the tip moves, so index and try again. An update
-    // that cannot advance the tip never will here, and no listener is bound yet.
-    while !Mempool::update(&mempool, &daemon, &tip)? {
-        let new_tip = indexer.update(&daemon)?;
-        if new_tip == tip {
-            warn!(
-                "index could not advance, starting up with a partial index tip='{}'",
-                tip
-            );
-            break;
+    let mut mempool_sync_retries = 0;
+    loop {
+        match Mempool::update(&mempool, &daemon, &tip)? {
+            MempoolSyncStatus::Synced => break,
+            MempoolSyncStatus::TipMoved => {
+                let new_tip = indexer.update(&daemon)?;
+                if new_tip == tip {
+                    warn!(
+                        "index could not advance, starting up with a partial index tip='{}'",
+                        tip
+                    );
+                    break;
+                }
+                tip = new_tip;
+            }
+            MempoolSyncStatus::FailedToIndex => {
+                mempool_sync_retries += 1;
+                if mempool_sync_retries >= MAX_INITIAL_MEMPOOL_SYNC_RETRIES {
+                    bail!(
+                        "initial mempool sync failed to index {} times, giving up",
+                        MAX_INITIAL_MEMPOOL_SYNC_RETRIES
+                    );
+                }
+                signal.wait(Duration::from_secs(3), false)?;
+            }
         }
-        tip = new_tip;
     }
 
     #[cfg(feature = "liquid")]
@@ -168,6 +192,7 @@ fn run_server(config: Arc<Config>, salt_rwlock: Arc<RwLock<String>>) -> Result<(
         "count of iterations of electrs main loop each 5 seconds or after interrupts",
     ));
 
+    let mut mempool_sync_failures = 0;
     loop {
         main_loop_count.inc();
 
@@ -185,8 +210,24 @@ fn run_server(config: Arc<Config>, salt_rwlock: Arc<RwLock<String>>) -> Result<(
         };
 
         // Update mempool
-        if !Mempool::update(&mempool, &daemon, &tip)? {
-            warn!("skipped failed mempool update, trying again in 5 seconds");
+        match Mempool::update(&mempool, &daemon, &tip)? {
+            MempoolSyncStatus::Synced => mempool_sync_failures = 0,
+            MempoolSyncStatus::TipMoved => {
+                warn!("mempool sync aborted: chain tip moved, retrying next cycle");
+            }
+            MempoolSyncStatus::FailedToIndex => {
+                mempool_sync_failures += 1;
+                if mempool_sync_failures >= MAX_CONSECUTIVE_MEMPOOL_SYNC_FAILURES {
+                    bail!(
+                        "mempool sync failed to index {} consecutive times, giving up",
+                        MAX_CONSECUTIVE_MEMPOOL_SYNC_FAILURES
+                    );
+                }
+                warn!(
+                    "mempool_sync_failed_to_index: retrying next cycle failures='{}'",
+                    mempool_sync_failures
+                );
+            }
         }
 
         // Update subscribed clients

src/chain.rs

@@ -2,7 +2,7 @@
 pub use bitcoin::{
     address, blockdata::block::Header as BlockHeader, blockdata::script, consensus::deserialize,
     hash_types::TxMerkleNode, Address, Block, BlockHash, OutPoint, ScriptBuf as Script, Sequence,
-    Transaction, TxIn, TxOut, Txid,
+    Transaction, TxIn, TxOut, Txid, Wtxid,
 };
 
 #[cfg(feature = "liquid")]
@@ -11,6 +11,7 @@ pub use {
     elements::{
         address, confidential, encode::deserialize, script, Address, AssetId, Block, BlockHash,
         BlockHeader, OutPoint, Script, Sequence, Transaction, TxIn, TxMerkleNode, TxOut, Txid,
+        Wtxid,
     },
 };