From 271ab5b463aaafccd6a5fdbcafa92383834146d3 Mon Sep 17 00:00:00 2001 From: evan-forbes Date: Thu, 20 Aug 2026 04:09:53 -0500 Subject: [PATCH 01/22] feat(mining): advertise admitted blocks optimistically --- crates/zakura-consensus/src/block.rs | 103 +++++-- crates/zakura-consensus/src/block/prepared.rs | 270 ++++++++++++++++++ crates/zakura-consensus/src/block/request.rs | 51 +++- crates/zakura-rpc/src/config/mining.rs | 32 ++- crates/zakura-rpc/src/lib.rs | 2 +- crates/zakura-rpc/src/methods.rs | 214 ++++++++++++-- .../zakura-rpc/src/methods/tests/snapshot.rs | 12 +- .../get_block_template_basic@mainnet_10.snap | 3 +- .../get_block_template_basic@testnet_10.snap | 3 +- ...t_block_template_long_poll@mainnet_10.snap | 1 + ...t_block_template_long_poll@testnet_10.snap | 1 + .../zakura-rpc/src/methods/tests/vectors.rs | 2 + .../src/methods/types/get_block_template.rs | 69 +++-- .../types/get_block_template/parameters.rs | 3 +- .../src/methods/types/submit_block.rs | 209 +++++++++++++- crates/zakura-rpc/src/server/error.rs | 21 -- .../zakura-rpc/tests/serialization_tests.rs | 2 + crates/zakura-state/src/lib.rs | 2 +- crates/zakura-state/src/request.rs | 92 +++++- crates/zakura-state/src/service.rs | 66 ++++- .../zakura-state/src/service/queued_blocks.rs | 11 +- .../service/queued_blocks/tests/vectors.rs | 6 +- crates/zakura-state/src/service/tests.rs | 6 +- crates/zakura-state/src/service/write.rs | 2 +- crates/zakurad/src/commands/start.rs | 7 +- crates/zakurad/src/components/inbound.rs | 77 ++++- .../components/inbound/tests/real_peer_set.rs | 12 +- crates/zakurad/src/components/sync/gossip.rs | 109 +++++-- .../src/components/sync/tests/gossip.rs | 28 +- crates/zakurad/tests/acceptance.rs | 15 +- 30 files changed, 1260 insertions(+), 171 deletions(-) create mode 100644 crates/zakura-consensus/src/block/prepared.rs diff --git a/crates/zakura-consensus/src/block.rs b/crates/zakura-consensus/src/block.rs index 565db91780..47efd06e52 100644 --- a/crates/zakura-consensus/src/block.rs +++ b/crates/zakura-consensus/src/block.rs @@ -34,6 +34,7 @@ use zakura_state as zs; use crate::{error::*, primitives, transaction as tx, BoxError}; pub mod check; +mod prepared; pub mod request; pub mod subsidy; @@ -49,6 +50,7 @@ pub struct SemanticBlockVerifier { network: Network, state_service: S, transaction_verifier: V, + prepared_candidates: prepared::PreparedCandidateCache, } /// Block verification errors. @@ -231,6 +233,7 @@ where network: network.clone(), state_service, transaction_verifier, + prepared_candidates: Default::default(), } } } @@ -258,6 +261,7 @@ where let mut state_service = self.state_service.clone(); let mut transaction_verifier = self.transaction_verifier.clone(); let network = self.network.clone(); + let prepared_candidates = self.prepared_candidates.clone(); let block = request.block(); @@ -267,6 +271,7 @@ where async move { let hash = zakura_header_chain::validate_encoding_version_hash(&block.header) .map_err(BlockError::from)?; + let preparation_start = request.should_cache().then(std::time::Instant::now); // Check that this block is actually a new block. tracing::trace!("checking that block is not already in state"); match state_service @@ -295,6 +300,36 @@ where Err(BlockError::MaxHeight(height, hash, block::Height::MAX))?; } + if request.is_mined_commit() { + let solved_header_start = std::time::Instant::now(); + if let Some(mut prepared_block) = + prepared_candidates.lookup(&block, request.work_id(), &network) + { + check::difficulty_is_valid(&block.header, &network, &height, &hash)?; + check::equihash_solution_is_valid(&block.header, &network)?; + check::time_is_valid_at(&block.header, Utc::now(), &height, &hash) + .map_err(VerifyBlockError::Time)?; + for transaction in &block.transactions { + tx::check::lock_time_has_passed(transaction, height, block.header.time) + .map_err(VerifyBlockError::Transaction)?; + } + metrics::histogram!("mining.solved_header_check.duration_seconds") + .record(solved_header_start.elapsed().as_secs_f64()); + + prepared_block.block = block; + prepared_block.hash = hash; + prepared_block.height = height; + return commit_prepared_block( + state_service, + prepared_block, + request.admission(), + ) + .await; + } + metrics::histogram!("mining.solved_header_check.duration_seconds") + .record(solved_header_start.elapsed().as_secs_f64()); + } + // > The block data MUST be validated and checked against the server's usual // > acceptance rules (excluding the check for a valid proof-of-work). // @@ -443,7 +478,8 @@ where // Return early for proposal requests. if request.is_proposal() { - return match state_service + let cache_copy = prepared_block.clone(); + let response = match state_service .ready() .await .map_err(VerifyBlockError::ValidateProposal)? @@ -454,26 +490,59 @@ where zs::Response::ValidBlockProposal => Ok(hash), _ => unreachable!("wrong response for CheckBlockProposalValidity"), }; - } - - match state_service - .ready() - .await - .map_err(|source| VerifyBlockError::StateService { source, hash })? - .call(zs::Request::CommitSemanticallyVerifiedBlock(prepared_block)) - .await - { - Ok(zs::Response::Committed(committed_hash)) => { - assert_eq!(committed_hash, hash, "state must commit correct hash"); - Ok(hash) + if response.is_ok() && request.should_cache() { + let candidate = cache_copy.block.clone(); + prepared_candidates.insert(&candidate, request.work_id(), cache_copy, &network); + metrics::histogram!("mining.preparation.duration_seconds").record( + preparation_start + .expect("cached preparation records its start time") + .elapsed() + .as_secs_f64(), + ); } - - Err(source) => Err(map_commit_error(source, hash)), - - _ => unreachable!("wrong response for CommitSemanticallyVerifiedBlock"), + return response; } + + commit_prepared_block(state_service, prepared_block, request.admission()).await } .instrument(span) .boxed() } } + +async fn commit_prepared_block( + mut state_service: S, + prepared_block: zs::SemanticallyVerifiedBlock, + admission: Option, +) -> Result +where + S: Service + Send + Clone + 'static, + S::Future: Send + 'static, +{ + let hash = prepared_block.hash; + let request = match admission { + Some(admission) => zs::Request::CommitSemanticallyVerifiedBlockWithAdmission { + block: prepared_block, + admission, + }, + None => zs::Request::CommitSemanticallyVerifiedBlock(prepared_block), + }; + let commit_start = std::time::Instant::now(); + let response = state_service + .ready() + .await + .map_err(|source| VerifyBlockError::StateService { source, hash })? + .call(request) + .await; + metrics::histogram!("mining.contextual_commit.duration_seconds") + .record(commit_start.elapsed().as_secs_f64()); + + match response { + Ok(zs::Response::Committed(committed_hash)) => { + assert_eq!(committed_hash, hash, "state must commit correct hash"); + Ok(hash) + } + Err(source) => Err(map_commit_error(source, hash)), + _ => unreachable!("wrong response for semantic block commit"), + } +} diff --git a/crates/zakura-consensus/src/block/prepared.rs b/crates/zakura-consensus/src/block/prepared.rs new file mode 100644 index 0000000000..b0a8b899d8 --- /dev/null +++ b/crates/zakura-consensus/src/block/prepared.rs @@ -0,0 +1,270 @@ +use std::{ + collections::VecDeque, + sync::{Arc, Mutex}, + time::{Duration, Instant}, +}; + +use blake2b_simd::Params; +use chrono::{DateTime, Utc}; +use zakura_chain::{ + block::{Block, Header}, + parameters::Network, + serialization::ZcashSerialize, + work::equihash::Solution, +}; +use zakura_state::SemanticallyVerifiedBlock; + +const MAX_ENTRIES: usize = 32; +const MAX_BYTES: usize = 64 * 1024 * 1024; +const ENTRY_TTL: Duration = Duration::from_secs(10 * 60); + +#[derive(Clone, Default)] +pub(super) struct PreparedCandidateCache(Arc>); + +impl std::fmt::Debug for PreparedCandidateCache { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let inner = self + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + f.debug_struct("PreparedCandidateCache") + .field("entries", &inner.entries.len()) + .field("bytes", &inner.bytes) + .finish() + } +} + +#[derive(Default)] +struct CacheInner { + entries: VecDeque, + bytes: usize, +} + +struct Entry { + work_id: Option, + fingerprint: [u8; 32], + immutable_bytes: Vec, + prepared: SemanticallyVerifiedBlock, + size: usize, + expires_at: Instant, +} + +impl PreparedCandidateCache { + pub(super) fn lookup( + &self, + block: &Block, + work_id: Option<&str>, + network: &Network, + ) -> Option { + let immutable_bytes = immutable_candidate_bytes(block, network); + let fingerprint = fingerprint(&immutable_bytes); + let mut inner = self + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + inner.prune_expired(); + + if let Some(work_id) = work_id { + if let Some(entry) = inner + .entries + .iter() + .find(|entry| entry.work_id.as_deref() == Some(work_id)) + { + if entry.immutable_bytes == immutable_bytes { + metrics::counter!("mining.prepared_cache.hits").increment(1); + return Some(entry.prepared.clone()); + } + + metrics::counter!("mining.prepared_cache.mismatches").increment(1); + return None; + } + } + + if let Some(entry) = inner.entries.iter().find(|entry| { + entry.fingerprint == fingerprint && entry.immutable_bytes == immutable_bytes + }) { + metrics::counter!("mining.prepared_cache.hits").increment(1); + return Some(entry.prepared.clone()); + } + + metrics::counter!("mining.prepared_cache.misses").increment(1); + None + } + + pub(super) fn insert( + &self, + block: &Block, + work_id: Option<&str>, + prepared: SemanticallyVerifiedBlock, + network: &Network, + ) { + let immutable_bytes = immutable_candidate_bytes(block, network); + let fingerprint = fingerprint(&immutable_bytes); + // Count the canonical candidate and the derived verification inputs conservatively. + let size = immutable_bytes.len().saturating_mul(2); + if size > MAX_BYTES { + return; + } + + let mut inner = self + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + inner.prune_expired(); + let work_id = work_id.map(ToOwned::to_owned).or_else(|| { + inner + .entries + .iter() + .find(|entry| { + entry.fingerprint == fingerprint && entry.immutable_bytes == immutable_bytes + }) + .and_then(|entry| entry.work_id.clone()) + }); + inner.remove_matching(fingerprint, &immutable_bytes); + + while inner.entries.len() >= MAX_ENTRIES || inner.bytes.saturating_add(size) > MAX_BYTES { + if !inner.evict_oldest() { + break; + } + } + + inner.bytes = inner.bytes.saturating_add(size); + inner.entries.push_back(Entry { + work_id, + fingerprint, + immutable_bytes, + prepared, + size, + expires_at: Instant::now() + ENTRY_TTL, + }); + } +} + +impl CacheInner { + fn prune_expired(&mut self) { + let now = Instant::now(); + while self + .entries + .front() + .is_some_and(|entry| entry.expires_at <= now) + { + self.evict_oldest(); + } + } + + fn remove_matching(&mut self, fingerprint: [u8; 32], immutable_bytes: &[u8]) { + if let Some(index) = self.entries.iter().position(|entry| { + entry.fingerprint == fingerprint && entry.immutable_bytes == immutable_bytes + }) { + let entry = self + .entries + .remove(index) + .expect("entry exists because its index came from the same deque"); + self.bytes = self.bytes.saturating_sub(entry.size); + } + } + + fn evict_oldest(&mut self) -> bool { + let Some(entry) = self.entries.pop_front() else { + return false; + }; + self.bytes = self.bytes.saturating_sub(entry.size); + metrics::counter!("mining.prepared_cache.evictions").increment(1); + true + } +} + +fn immutable_candidate_bytes(block: &Block, network: &Network) -> Vec { + let mut header: Header = *block.header; + header.time = + DateTime::::from_timestamp(0, 0).expect("the Unix epoch is a valid UTC timestamp"); + header.nonce = [0; 32].into(); + header.solution = Solution::for_proposal_for_network(network); + + Block { + header: Arc::new(header), + transactions: block.transactions.clone(), + } + .zcash_serialize_to_vec() + .expect("serialization to memory cannot fail") +} + +fn fingerprint(bytes: &[u8]) -> [u8; 32] { + let hash = Params::new().hash_length(32).hash(bytes); + let mut fingerprint = [0; 32]; + fingerprint.copy_from_slice(hash.as_bytes()); + fingerprint +} + +#[cfg(test)] +mod tests { + use super::*; + use zakura_chain::{ + block::Hash, serialization::ZcashDeserialize, work::difficulty::INVALID_COMPACT_DIFFICULTY, + }; + + fn test_block() -> Block { + Block::zcash_deserialize(&zakura_test::vectors::BLOCK_MAINNET_GENESIS_BYTES[..]) + .expect("the genesis test vector is valid") + } + + #[test] + fn solved_header_fields_reuse_prepared_candidate() { + let network = Network::Mainnet; + let original = test_block(); + let prepared = SemanticallyVerifiedBlock::from(Arc::new(original.clone())); + let cache = PreparedCandidateCache::default(); + cache.insert(&original, Some("work"), prepared, &network); + + let mut solved = original; + let header = Arc::make_mut(&mut solved.header); + header.nonce = [7; 32].into(); + header.solution = Solution::for_proposal_for_network(&network); + header.time += chrono::Duration::seconds(1); + + assert!(cache.lookup(&solved, Some("work"), &network).is_some()); + assert!(cache.lookup(&solved, None, &network).is_some()); + } + + #[test] + fn immutable_candidate_changes_do_not_reuse_work_id() { + let network = Network::Mainnet; + let original = test_block(); + let prepared = SemanticallyVerifiedBlock::from(Arc::new(original.clone())); + let cache = PreparedCandidateCache::default(); + cache.insert(&original, Some("work"), prepared, &network); + + let mut changed_parent = original.clone(); + Arc::make_mut(&mut changed_parent.header).previous_block_hash = Hash([1; 32]); + assert!(cache + .lookup(&changed_parent, Some("work"), &network) + .is_none()); + + let mut changed_header = original.clone(); + Arc::make_mut(&mut changed_header.header).version ^= 1; + assert!(cache + .lookup(&changed_header, Some("work"), &network) + .is_none()); + + let mut changed_commitment = original.clone(); + Arc::make_mut(&mut changed_commitment.header).commitment_bytes[0] ^= 1; + assert!(cache + .lookup(&changed_commitment, Some("work"), &network) + .is_none()); + + let mut changed_difficulty = original.clone(); + Arc::make_mut(&mut changed_difficulty.header).difficulty_threshold = + INVALID_COMPACT_DIFFICULTY; + assert!(cache + .lookup(&changed_difficulty, Some("work"), &network) + .is_none()); + + let mut changed_transactions = original; + changed_transactions + .transactions + .push(changed_transactions.transactions[0].clone()); + assert!(cache + .lookup(&changed_transactions, Some("work"), &network) + .is_none()); + } +} diff --git a/crates/zakura-consensus/src/block/request.rs b/crates/zakura-consensus/src/block/request.rs index 773f4443e9..780a3e7dfe 100644 --- a/crates/zakura-consensus/src/block/request.rs +++ b/crates/zakura-consensus/src/block/request.rs @@ -3,16 +3,33 @@ use std::sync::Arc; use zakura_chain::block::Block; +use zakura_state::BlockAdmission; #[derive(Debug, Clone, PartialEq, Eq)] /// A request to the chain or block verifier pub enum Request { /// Performs semantic validation, then asks the state to perform contextual validation and commit the block Commit(Arc), + /// Reuses prepared mining work when possible, then commits the solved block. + CommitMined { + /// The solved block. + block: Arc, + /// The template work ID supplied by the miner. + work_id: Option, + /// State write-queue admission notification. + admission: BlockAdmission, + }, /// Performs semantic validation but skips checking proof of work, /// then asks the state to perform contextual validation. /// Does not commit the block to the state. CheckProposal(Arc), + /// Validates and caches a mining candidate without checking proof of work. + Prepare { + /// The unsolved candidate block. + block: Arc, + /// The template work ID, when one was assigned. + work_id: Option, + }, } impl Request { @@ -20,15 +37,45 @@ impl Request { pub fn block(&self) -> Arc { Arc::clone(match self { Request::Commit(block) => block, + Request::CommitMined { block, .. } => block, Request::CheckProposal(block) => block, + Request::Prepare { block, .. } => block, }) } /// Returns `true` if the request is a proposal pub fn is_proposal(&self) -> bool { match self { - Request::Commit(_) => false, - Request::CheckProposal(_) => true, + Request::Commit(_) | Request::CommitMined { .. } => false, + Request::CheckProposal(_) | Request::Prepare { .. } => true, } } + + /// Returns true when a successful proposal should populate the prepared-candidate cache. + pub fn should_cache(&self) -> bool { + matches!(self, Request::Prepare { .. }) + } + + /// Returns the supplied mining work ID. + pub fn work_id(&self) -> Option<&str> { + match self { + Request::CommitMined { work_id, .. } | Request::Prepare { work_id, .. } => { + work_id.as_deref() + } + Request::Commit(_) | Request::CheckProposal(_) => None, + } + } + + /// Returns the state admission notification for a mined commit. + pub fn admission(&self) -> Option { + match self { + Request::CommitMined { admission, .. } => Some(admission.clone()), + _ => None, + } + } + + /// Returns true for a mined-block commit. + pub fn is_mined_commit(&self) -> bool { + matches!(self, Request::CommitMined { .. }) + } } diff --git a/crates/zakura-rpc/src/config/mining.rs b/crates/zakura-rpc/src/config/mining.rs index 15583080d5..97a82ab0b3 100644 --- a/crates/zakura-rpc/src/config/mining.rs +++ b/crates/zakura-rpc/src/config/mining.rs @@ -34,7 +34,7 @@ pub(crate) const MAX_USER_COINBASE_DATA_LEN: usize = /// Mining configuration section. #[serde_as] -#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] +#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)] #[serde(deny_unknown_fields, default)] pub struct Config { /// Address for receiving miner subsidy and tx fees. @@ -63,6 +63,21 @@ pub struct Config { /// The internal miner is off by default. #[serde(default)] pub internal_miner: bool, + + /// Advertise mined block hashes after state admission and before contextual commit completes. + pub optimistic_block_inventory: bool, +} + +impl Default for Config { + fn default() -> Self { + Self { + miner_address: None, + extra_coinbase_data: None, + miner_memo: None, + internal_miner: false, + optimistic_block_inventory: true, + } + } } impl Config { @@ -173,3 +188,18 @@ lazy_static::lazy_static! { ].into()), ].into(); } + +#[cfg(test)] +mod tests { + use super::Config; + + #[test] + fn optimistic_block_inventory_defaults_on_and_can_be_disabled() { + let default: Config = toml::from_str("").expect("empty mining config uses defaults"); + assert!(default.optimistic_block_inventory); + + let disabled: Config = toml::from_str("optimistic_block_inventory = false") + .expect("the optimistic inventory option is valid"); + assert!(!disabled.optimistic_block_inventory); + } +} diff --git a/crates/zakura-rpc/src/lib.rs b/crates/zakura-rpc/src/lib.rs index 81377ce41d..67022674cd 100644 --- a/crates/zakura-rpc/src/lib.rs +++ b/crates/zakura-rpc/src/lib.rs @@ -20,5 +20,5 @@ mod tests; pub use methods::types::{ get_block_template::{fetch_chain_info, proposal::proposal_block_from_template, MinerParams}, - submit_block::SubmitBlockChannel, + submit_block::{MinedBlockEvent, PendingBlockRegistry, SubmitBlockChannel}, }; diff --git a/crates/zakura-rpc/src/methods.rs b/crates/zakura-rpc/src/methods.rs index 06addd54c7..006159c80f 100644 --- a/crates/zakura-rpc/src/methods.rs +++ b/crates/zakura-rpc/src/methods.rs @@ -130,7 +130,10 @@ use types::{ long_poll::LongPollInput, network_info::{GetNetworkInfoResponse, NetworkInfo}, peer_info::PeerInfo, - submit_block::{SubmitBlockErrorResponse, SubmitBlockParameters, SubmitBlockResponse}, + submit_block::{ + MinedBlockEvent, PendingBlockRegistry, SubmitBlockErrorResponse, SubmitBlockParameters, + SubmitBlockResponse, + }, subsidy::GetBlockSubsidyResponse, transaction::TransactionObject, unified_address::ZListUnifiedReceiversResponse, @@ -576,8 +579,7 @@ pub trait Rpc { /// /// # Notes /// - /// Arguments to this RPC are currently ignored. - /// Long polling, block proposals, server lists, and work IDs are not supported. + /// Server lists are not supported. Long polling, block proposals, and work IDs are supported. /// /// Miners can make arbitrary changes to blocks, as long as: /// - the data sent to `submitblock` is a valid Zcash block, and @@ -601,7 +603,7 @@ pub trait Rpc { /// # Parameters /// /// - `hexdata`: (string, required) - /// - `jsonparametersobject`: (string, optional) - currently ignored + /// - `jsonparametersobject`: (string, optional) /// /// # Notes /// @@ -943,7 +945,49 @@ where latest_chain_tip: Tip, address_book: AddressBook, last_warn_error_log_rx: LoggedLastEvent, - mined_block_sender: Option>, + mined_block_sender: Option>, + ) -> (Self, JoinHandle<()>) + where + VersionString: ToString + Clone + Send + 'static, + UserAgentString: ToString + Clone + Send + 'static, + { + Self::new_with_pending_blocks( + network, + mining_config, + debug_force_finished_sync, + build_version, + user_agent, + mempool, + state, + read_state, + block_verifier_router, + sync_status, + latest_chain_tip, + address_book, + last_warn_error_log_rx, + mined_block_sender, + PendingBlockRegistry::default(), + ) + } + + /// Creates an RPC handler with a pending-block registry shared with peer serving. + #[allow(clippy::too_many_arguments)] + pub fn new_with_pending_blocks( + network: Network, + mining_config: config::mining::Config, + debug_force_finished_sync: bool, + build_version: VersionString, + user_agent: UserAgentString, + mempool: Mempool, + state: State, + read_state: ReadState, + block_verifier_router: BlockVerifierRouter, + sync_status: SyncStatus, + latest_chain_tip: Tip, + address_book: AddressBook, + last_warn_error_log_rx: LoggedLastEvent, + mined_block_sender: Option>, + pending_blocks: PendingBlockRegistry, ) -> (Self, JoinHandle<()>) where VersionString: ToString + Clone + Send + 'static, @@ -959,12 +1003,13 @@ where build_version.insert(0, 'v'); } - let gbt = GetBlockTemplateHandler::new( + let gbt = GetBlockTemplateHandler::new_with_pending_blocks( &network, mining_config.clone(), block_verifier_router, sync_status, mined_block_sender, + pending_blocks, ); let rpc_impl = RpcImpl { @@ -998,6 +1043,34 @@ where &self.network } + fn prepare_template_in_background(&self, template: &BlockTemplateResponse) { + #[cfg(test)] + { + let _ = template; + return; + } + + #[cfg(not(test))] + { + let Ok(block) = proposal_block_from_template(template, None, &self.network) else { + return; + }; + let request = zakura_consensus::Request::Prepare { + block: Arc::new(block), + work_id: Some(template.work_id().clone()), + }; + let verifier = self.gbt.block_verifier_router(); + tokio::spawn( + async move { + if let Err(error) = verifier.oneshot(request).await { + tracing::debug!(?error, "background mining candidate preparation failed"); + } + } + .in_current_span(), + ); + } + } + /// Sets the end-of-support height reported by `getdeprecationinfo`. /// /// When unset, or set to `None`, the RPC omits `end_of_service`. @@ -2400,12 +2473,16 @@ where .as_ref() .and_then(GetBlockTemplateParameters::block_proposal_data) { + let work_id = parameters + .as_ref() + .and_then(|parameters| parameters.work_id.clone()); return validate_block_proposal( self.gbt.block_verifier_router(), block_proposal_bytes, &self.network, latest_chain_tip, sync_status, + work_id, ) .await; } @@ -2624,7 +2701,7 @@ where // Respond instantly with an empty block upon a chain tip change so that // the miner doesn't waste their effort trying to extend a shorter // chain. - return Ok(BlockTemplateResponse::new_internal( + let template = BlockTemplateResponse::new_internal( &self.network, precomputed_coinbase, miner_params, @@ -2632,8 +2709,9 @@ where server_long_poll_id, vec![], submit_old, - ) - .into()) + ); + self.prepare_template_in_background(&template); + return Ok(template.into()) } // The max time does not elapse during normal operation on mainnet, @@ -2688,7 +2766,7 @@ where // - After this point, the template only depends on the previously fetched data. - Ok(BlockTemplateResponse::new_internal( + let template = BlockTemplateResponse::new_internal( &self.network, None, miner_params, @@ -2696,16 +2774,18 @@ where server_long_poll_id, mempool_txs, submit_old, - ) - .into()) + ); + self.prepare_template_in_background(&template); + Ok(template.into()) } async fn submit_block( &self, HexData(block_bytes): HexData, - _parameters: Option, + parameters: Option, ) -> Result { let mut block_verifier_router = self.gbt.block_verifier_router(); + let submitted_at = std::time::Instant::now(); let block: Block = match block_bytes.zcash_deserialize_into() { Ok(block_bytes) => block_bytes, @@ -2723,13 +2803,104 @@ where .coinbase_height() .ok_or_error(0, "coinbase height not found")?; let block_hash = block.hash(); + let block = Arc::new(block); + let work_id = parameters.and_then(|parameters| parameters.work_id); + let admission = zakura_state::BlockAdmission::pending(); + let request = zakura_consensus::Request::CommitMined { + block: block.clone(), + work_id, + admission: admission.clone(), + }; + let pending_blocks = self.gbt.pending_blocks(); + let mined_block_sender = self.gbt.mined_block_sender(); + let optimistic_block_inventory = self.gbt.optimistic_block_inventory(); + + // This task owns the commit and registry lifecycle. RPC cancellation only detaches it. + let lifecycle = tokio::spawn(async move { + let verification = + async move { block_verifier_router.ready().await?.call(request).await }; + tokio::pin!(verification); + + let admission_start = std::time::Instant::now(); + let mut early_result = None; + let verification_result = tokio::select! { + biased; - let block_verifier_router_response = block_verifier_router - .ready() - .await - .map_error(0)? - .call(zakura_consensus::Request::Commit(Arc::new(block))) - .await; + admitted = admission.wait() => { + metrics::histogram!("mining.state_admission.duration_seconds") + .record(admission_start.elapsed().as_secs_f64()); + if admitted && optimistic_block_inventory && pending_blocks.insert(block.clone()) { + let (advertised, receiver) = tokio::sync::oneshot::channel(); + let event = MinedBlockEvent::Early { + hash: block_hash, + height, + submitted_at, + advertised, + }; + if mined_block_sender.try_send(event).is_ok() { + early_result = Some(receiver); + } else { + metrics::counter!("mining.optimistic_inventory.fallbacks").increment(1); + pending_blocks.resolve(block_hash, Err(())); + } + } + verification.await + }, + result = &mut verification => result, + }; + + pending_blocks.resolve( + block_hash, + verification_result + .as_ref() + .map(|_| block.clone()) + .map_err(|_| ()), + ); + + let committed = verification_result.is_ok(); + tokio::spawn(async move { + let early_advertised = match early_result { + Some(receiver) => tokio::time::timeout(Duration::from_secs(20), receiver) + .await + .ok() + .and_then(|result| result.ok()) + .unwrap_or(false), + None => false, + }; + let event = if committed { + MinedBlockEvent::Committed { + hash: block_hash, + height, + early_advertised, + } + } else { + if early_advertised { + metrics::counter!("mining.optimistic_inventory.post_commit_failures") + .increment(1); + tracing::warn!( + ?block_hash, + ?height, + "mined block failed contextual commit after early inventory" + ); + } + MinedBlockEvent::Failed { + hash: block_hash, + height, + early_advertised, + } + }; + let _ = mined_block_sender.send(event).await; + }); + verification_result + }); + + let block_verifier_router_response = lifecycle.await.map_err(|error| { + ErrorObject::owned( + ErrorCode::InternalError.code(), + format!("mined block lifecycle task failed: {error}"), + None::<()>, + ) + })?; let chain_error = match block_verifier_router_response { // Currently, this match arm returns `null` (Accepted) for blocks committed @@ -2740,11 +2911,6 @@ where // The difference is important to miners, because they want to mine on the best chain. Ok(hash) => { tracing::info!(?hash, ?height, "submit block accepted"); - - self.gbt - .advertise_mined_block(hash, height) - .map_error_with_prefix(0, "failed to send mined block to gossip task")?; - return Ok(SubmitBlockResponse::Accepted); } diff --git a/crates/zakura-rpc/src/methods/tests/snapshot.rs b/crates/zakura-rpc/src/methods/tests/snapshot.rs index 5d05edac44..207eca66d3 100644 --- a/crates/zakura-rpc/src/methods/tests/snapshot.rs +++ b/crates/zakura-rpc/src/methods/tests/snapshot.rs @@ -867,7 +867,14 @@ fn snapshot_rpc_getblocktemplate( settings: &insta::Settings, ) { settings.bind(|| { - insta::assert_json_snapshot!(format!("get_block_template_{variant}"), block_template) + insta::assert_json_snapshot!(format!("get_block_template_{variant}"), block_template, { + ".workid" => dynamic_redaction(|value, _path| { + let work_id = value.as_str().expect("workid must be a string"); + assert_eq!(work_id.len(), 32, "workid must encode 16 bytes"); + assert!(work_id.bytes().all(|byte| byte.is_ascii_hexdigit())); + "[WorkId]" + }), + }) }); if let Some(coinbase_tx) = coinbase_tx { @@ -1020,6 +1027,7 @@ pub async fn test_mining_rpcs( miner_memo: None, // TODO: Use default field values when optional features are enabled in tests #8183 internal_miner: true, + optimistic_block_inventory: true, }; // nu5 block height @@ -1347,7 +1355,7 @@ pub async fn test_mining_rpcs( let mock_block_verifier_router_request_handler = async move { mock_block_verifier_router - .expect_request_that(|req| matches!(req, zakura_consensus::Request::CheckProposal(_))) + .expect_request_that(|req| matches!(req, zakura_consensus::Request::Prepare { .. })) .await .respond(Hash::from([0; 32])); }; diff --git a/crates/zakura-rpc/src/methods/tests/snapshots/get_block_template_basic@mainnet_10.snap b/crates/zakura-rpc/src/methods/tests/snapshots/get_block_template_basic@mainnet_10.snap index 89847107af..0bdee09653 100644 --- a/crates/zakura-rpc/src/methods/tests/snapshots/get_block_template_basic@mainnet_10.snap +++ b/crates/zakura-rpc/src/methods/tests/snapshots/get_block_template_basic@mainnet_10.snap @@ -41,5 +41,6 @@ expression: block_template "curtime": 1654008617, "bits": "1f055554", "height": 1687105, - "maxtime": 1654008728 + "maxtime": 1654008728, + "workid": "[WorkId]" } diff --git a/crates/zakura-rpc/src/methods/tests/snapshots/get_block_template_basic@testnet_10.snap b/crates/zakura-rpc/src/methods/tests/snapshots/get_block_template_basic@testnet_10.snap index 567b6a5877..d5dae7fc85 100644 --- a/crates/zakura-rpc/src/methods/tests/snapshots/get_block_template_basic@testnet_10.snap +++ b/crates/zakura-rpc/src/methods/tests/snapshots/get_block_template_basic@testnet_10.snap @@ -41,5 +41,6 @@ expression: block_template "curtime": 1654008617, "bits": "20055554", "height": 1842421, - "maxtime": 1654008728 + "maxtime": 1654008728, + "workid": "[WorkId]" } diff --git a/crates/zakura-rpc/src/methods/tests/snapshots/get_block_template_long_poll@mainnet_10.snap b/crates/zakura-rpc/src/methods/tests/snapshots/get_block_template_long_poll@mainnet_10.snap index 1cb709d0a3..85e72524f1 100644 --- a/crates/zakura-rpc/src/methods/tests/snapshots/get_block_template_long_poll@mainnet_10.snap +++ b/crates/zakura-rpc/src/methods/tests/snapshots/get_block_template_long_poll@mainnet_10.snap @@ -42,5 +42,6 @@ expression: block_template "bits": "1f055554", "height": 1687105, "maxtime": 1654008728, + "workid": "[WorkId]", "submitold": false } diff --git a/crates/zakura-rpc/src/methods/tests/snapshots/get_block_template_long_poll@testnet_10.snap b/crates/zakura-rpc/src/methods/tests/snapshots/get_block_template_long_poll@testnet_10.snap index 5738a1be32..421edca81c 100644 --- a/crates/zakura-rpc/src/methods/tests/snapshots/get_block_template_long_poll@testnet_10.snap +++ b/crates/zakura-rpc/src/methods/tests/snapshots/get_block_template_long_poll@testnet_10.snap @@ -42,5 +42,6 @@ expression: block_template "bits": "20055554", "height": 1842421, "maxtime": 1654008728, + "workid": "[WorkId]", "submitold": false } diff --git a/crates/zakura-rpc/src/methods/tests/vectors.rs b/crates/zakura-rpc/src/methods/tests/vectors.rs index f8d7d1e9a0..e07b9612c1 100644 --- a/crates/zakura-rpc/src/methods/tests/vectors.rs +++ b/crates/zakura-rpc/src/methods/tests/vectors.rs @@ -3093,6 +3093,7 @@ async fn gbt_with(net: Network, addr: ZcashAddress) { extra_coinbase_data: None, miner_memo: None, internal_miner: true, + optimistic_block_inventory: true, }; // nu5 block height @@ -3783,6 +3784,7 @@ async fn rpc_getdifficulty() { extra_coinbase_data: None, miner_memo: None, internal_miner: true, + optimistic_block_inventory: true, }; // nu5 block height diff --git a/crates/zakura-rpc/src/methods/types/get_block_template.rs b/crates/zakura-rpc/src/methods/types/get_block_template.rs index df4702fb3b..db6e6fe366 100644 --- a/crates/zakura-rpc/src/methods/types/get_block_template.rs +++ b/crates/zakura-rpc/src/methods/types/get_block_template.rs @@ -18,7 +18,7 @@ use derive_new::new; use jsonrpsee::core::RpcResult; use jsonrpsee_types::{ErrorCode, ErrorObject}; use rand::{rngs::OsRng, RngCore}; -use tokio::sync::mpsc::{self, error::TrySendError}; +use tokio::sync::mpsc; use tower::{Service, ServiceExt}; use zcash_keys::address::Address; use zcash_protocol::memo::MemoBytes; @@ -53,7 +53,7 @@ use crate::{ default_roots::DefaultRoots, long_poll::LongPollId, transaction::TransactionTemplate, }, server::error::OkOrError, - SubmitBlockChannel, + MinedBlockEvent, PendingBlockRegistry, SubmitBlockChannel, }; use constants::{ @@ -81,12 +81,8 @@ type InBlockTxDependenciesDepth = usize; pub struct BlockTemplateResponse { /// The getblocktemplate RPC capabilities supported by Zebra. /// - /// At the moment, Zebra does not support any of the extra capabilities from the specification: - /// - `proposal`: - /// - `longpoll`: - /// - `serverlist`: - /// - /// By the above, Zebra will always return an empty vector here. + /// Zakura accepts proposal, long-poll, and work-ID fields without requiring miners to declare + /// those capabilities. Zakura does not support server lists. pub(crate) capabilities: Vec, /// The version of the block format. @@ -204,6 +200,10 @@ pub struct BlockTemplateResponse { #[getter(copy)] pub(crate) max_time: DateTime32, + /// Identifies this prepared mining candidate. + #[serde(rename = "workid")] + pub(crate) work_id: String, + /// > only relevant for long poll responses: /// > indicates if work received prior to this response remains potentially valid (default) /// > and should have its shares submitted; @@ -254,6 +254,7 @@ impl fmt::Debug for BlockTemplateResponse { .field("bits", &self.bits) .field("height", &self.height) .field("max_time", &self.max_time) + .field("work_id", &self.work_id) .field("submit_old", &self.submit_old) .finish() } @@ -395,11 +396,19 @@ impl BlockTemplateResponse { max_time: chain_info.max_time, + work_id: new_work_id(), + submit_old, } } } +fn new_work_id() -> String { + let mut bytes = [0; 16]; + OsRng.fill_bytes(&mut bytes); + hex::encode(bytes) +} + #[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[serde(untagged)] /// A `getblocktemplate` RPC response. @@ -561,7 +570,13 @@ where /// A channel to send successful block submissions to the block gossip task, /// so they can be advertised to peers. - mined_block_sender: mpsc::Sender<(block::Hash, block::Height)>, + mined_block_sender: mpsc::Sender, + + /// Blocks whose hashes were advertised before contextual commit completed. + pending_blocks: PendingBlockRegistry, + + /// Whether state admission can trigger an early inventory. + optimistic_block_inventory: bool, } impl GetBlockTemplateHandler @@ -569,20 +584,24 @@ where BlockVerifierRouter: BlockVerifierService, SyncStatus: ChainSyncStatus + Clone + Send + Sync + 'static, { - /// Creates a new [`GetBlockTemplateHandler`]. - pub fn new( + /// Creates a handler with a registry shared by RPC and peer serving. + pub fn new_with_pending_blocks( net: &Network, conf: config::mining::Config, block_verifier_router: BlockVerifierRouter, sync_status: SyncStatus, - mined_block_sender: Option>, + mined_block_sender: Option>, + pending_blocks: PendingBlockRegistry, ) -> Self { + let optimistic_block_inventory = conf.optimistic_block_inventory; Self { miner_params: MinerParams::new(net, conf).ok(), block_verifier_router, sync_status, mined_block_sender: mined_block_sender .unwrap_or(SubmitBlockChannel::default().sender()), + pending_blocks, + optimistic_block_inventory, } } @@ -601,13 +620,19 @@ where self.block_verifier_router.clone() } - /// Advertises the mined block. - pub fn advertise_mined_block( - &self, - block: block::Hash, - height: block::Height, - ) -> Result<(), TrySendError<(block::Hash, block::Height)>> { - self.mined_block_sender.try_send((block, height)) + /// Returns a sender for the owned mined-block lifecycle task. + pub fn mined_block_sender(&self) -> mpsc::Sender { + self.mined_block_sender.clone() + } + + /// Returns the shared pending-block registry. + pub fn pending_blocks(&self) -> PendingBlockRegistry { + self.pending_blocks.clone() + } + + /// Returns whether early mined-block inventory is enabled. + pub fn optimistic_block_inventory(&self) -> bool { + self.optimistic_block_inventory } /// Randomizes the coinbase data, if miner parameters are set. @@ -689,6 +714,7 @@ pub async fn validate_block_proposal( net: &Network, latest_chain_tip: Tip, sync_status: SyncStatus, + work_id: Option, ) -> RpcResult where BlockVerifierRouter: Service< @@ -724,7 +750,10 @@ where .ready() .await .map_err(|error| ErrorObject::owned(0, error.to_string(), None::<()>))? - .call(zakura_consensus::Request::CheckProposal(Arc::new(block))) + .call(zakura_consensus::Request::Prepare { + block: Arc::new(block), + work_id, + }) .await; Ok(block_verifier_router_response diff --git a/crates/zakura-rpc/src/methods/types/get_block_template/parameters.rs b/crates/zakura-rpc/src/methods/types/get_block_template/parameters.rs index 7af639dfb7..3209622b3b 100644 --- a/crates/zakura-rpc/src/methods/types/get_block_template/parameters.rs +++ b/crates/zakura-rpc/src/methods/types/get_block_template/parameters.rs @@ -102,10 +102,9 @@ pub struct GetBlockTemplateParameters { /// The workid for the block template. /// - /// currently unused. #[serde(rename = "workid")] #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) _work_id: Option, + pub(crate) work_id: Option, } impl GetBlockTemplateParameters { diff --git a/crates/zakura-rpc/src/methods/types/submit_block.rs b/crates/zakura-rpc/src/methods/types/submit_block.rs index f347bb730d..9ea040ac5a 100644 --- a/crates/zakura-rpc/src/methods/types/submit_block.rs +++ b/crates/zakura-rpc/src/methods/types/submit_block.rs @@ -1,6 +1,12 @@ //! Parameter and response types for the `submitblock` RPC. -use tokio::sync::mpsc; +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, + time::Duration, +}; + +use tokio::sync::{mpsc, oneshot, watch}; use zakura_chain::block; @@ -13,7 +19,7 @@ use crate::methods::GetBlockTemplateHandler; /// See the notes for the [`submit_block`](crate::methods::RpcServer::submit_block) RPC. #[derive(Clone, Debug, PartialEq, Eq, serde::Deserialize, schemars::JsonSchema)] pub struct SubmitBlockParameters { - /// The workid for the block template. Currently unused. + /// The workid for the block template. /// /// > If the server provided a workid, it MUST be included with submissions, /// @@ -26,7 +32,134 @@ pub struct SubmitBlockParameters { /// /// #[serde(rename = "workid")] - pub _work_id: Option, + pub work_id: Option, +} + +/// The maximum time a peer waits for an early-advertised block to commit. +pub const PENDING_BLOCK_WAIT: Duration = Duration::from_secs(15); + +const MAX_PENDING_BLOCKS: usize = 16; + +/// A mined-block lifecycle event consumed by the block gossip task. +#[derive(Debug)] +pub enum MinedBlockEvent { + /// State admitted the block, so peers can receive its inventory before commit completes. + Early { + /// The block hash. + hash: block::Hash, + /// The block height. + height: block::Height, + /// When the RPC accepted the submitted bytes. + submitted_at: std::time::Instant, + /// Reports whether the early network advertisement completed. + advertised: oneshot::Sender, + }, + /// The contextual commit completed. + Committed { + /// The block hash. + hash: block::Hash, + /// The block height. + height: block::Height, + /// Whether the early advertisement completed successfully. + early_advertised: bool, + }, + /// The contextual commit failed after state admission. + Failed { + /// The block hash. + hash: block::Hash, + /// The block height. + height: block::Height, + /// Whether peers received an early inventory. + early_advertised: bool, + }, +} + +#[derive(Clone, Debug)] +enum PendingStatus { + Waiting, + Committed(Arc), + Failed, +} + +#[derive(Debug)] +struct PendingBlock { + status: watch::Sender, +} + +/// Holds early-advertised block bodies until their contextual commits finish. +#[derive(Clone, Debug, Default)] +pub struct PendingBlockRegistry(Arc>>); + +impl PendingBlockRegistry { + /// Inserts a block before its early inventory is sent. + /// + /// Returns false when the bounded registry is full. + pub fn insert(&self, block: Arc) -> bool { + let hash = block.hash(); + let mut entries = self + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if entries.contains_key(&hash) { + return false; + } + if entries.len() >= MAX_PENDING_BLOCKS { + metrics::counter!("mining.pending_registry.saturated").increment(1); + return false; + } + + let (status, _receiver) = watch::channel(PendingStatus::Waiting); + entries.insert(hash, PendingBlock { status }); + true + } + + /// Resolves peer waiters and removes a terminal entry. + pub fn resolve(&self, hash: block::Hash, result: Result, ()>) { + let entry = self + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&hash); + let Some(entry) = entry else { + return; + }; + let status = match result { + Ok(block) => PendingStatus::Committed(block), + Err(()) => PendingStatus::Failed, + }; + entry.status.send_replace(status); + } + + /// Waits for an early-advertised block to commit. + pub async fn wait(&self, hash: block::Hash) -> Option> { + let mut status = self + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(&hash) + .map(|entry| entry.status.subscribe())?; + let start = std::time::Instant::now(); + let result = tokio::time::timeout(PENDING_BLOCK_WAIT, async { + loop { + let current = status.borrow().clone(); + match current { + PendingStatus::Waiting => { + if status.changed().await.is_err() { + return None; + } + } + PendingStatus::Committed(block) => return Some(block), + PendingStatus::Failed => return None, + } + } + }) + .await + .ok() + .flatten(); + metrics::histogram!("mining.pending_peer_wait.duration_seconds") + .record(start.elapsed().as_secs_f64()); + result + } } /// Response to a `submitblock` RPC request. @@ -72,9 +205,9 @@ impl From for SubmitBlockResponse { /// A submit block channel, used to inform the gossip task about mined blocks. pub struct SubmitBlockChannel { /// The channel sender - sender: mpsc::Sender<(block::Hash, block::Height)>, + sender: mpsc::Sender, /// The channel receiver - receiver: mpsc::Receiver<(block::Hash, block::Height)>, + receiver: mpsc::Receiver, } impl SubmitBlockChannel { @@ -93,12 +226,12 @@ impl SubmitBlockChannel { } /// Get the channel sender - pub fn sender(&self) -> mpsc::Sender<(block::Hash, block::Height)> { + pub fn sender(&self) -> mpsc::Sender { self.sender.clone() } /// Get the channel receiver - pub fn receiver(self) -> mpsc::Receiver<(block::Hash, block::Height)> { + pub fn receiver(self) -> mpsc::Receiver { self.receiver } } @@ -108,3 +241,65 @@ impl Default for SubmitBlockChannel { Self::new() } } + +#[cfg(test)] +mod tests { + use super::*; + use zakura_chain::{block::Block, serialization::ZcashDeserializeInto}; + + fn test_block() -> Arc { + zakura_test::vectors::BLOCK_MAINNET_GENESIS_BYTES + .zcash_deserialize_into() + .expect("the genesis test vector is valid") + } + + #[tokio::test] + async fn pending_block_waits_for_success() { + let registry = PendingBlockRegistry::default(); + let block = test_block(); + let hash = block.hash(); + assert!(registry.insert(block.clone())); + + let wait = tokio::spawn({ + let registry = registry.clone(); + async move { registry.wait(hash).await } + }); + tokio::task::yield_now().await; + registry.resolve(hash, Ok(block.clone())); + + assert_eq!(wait.await.expect("wait task succeeds"), Some(block)); + } + + #[tokio::test] + async fn pending_block_failure_returns_not_found() { + let registry = PendingBlockRegistry::default(); + let block = test_block(); + let hash = block.hash(); + assert!(registry.insert(block)); + + let wait = tokio::spawn({ + let registry = registry.clone(); + async move { registry.wait(hash).await } + }); + tokio::task::yield_now().await; + registry.resolve(hash, Err(())); + + assert_eq!(wait.await.expect("wait task succeeds"), None); + } + + #[test] + fn pending_registry_is_bounded() { + let registry = PendingBlockRegistry::default(); + let original = test_block(); + for nonce in 0..MAX_PENDING_BLOCKS { + let mut block = (*original).clone(); + let nonce = u8::try_from(nonce).expect("the registry bound fits in u8"); + Arc::make_mut(&mut block.header).nonce = [nonce; 32].into(); + assert!(registry.insert(Arc::new(block))); + } + + let mut overflow = (*original).clone(); + Arc::make_mut(&mut overflow.header).nonce = [u8::MAX; 32].into(); + assert!(!registry.insert(Arc::new(overflow))); + } +} diff --git a/crates/zakura-rpc/src/server/error.rs b/crates/zakura-rpc/src/server/error.rs index 20bdf27aa7..835e3c4581 100644 --- a/crates/zakura-rpc/src/server/error.rs +++ b/crates/zakura-rpc/src/server/error.rs @@ -69,13 +69,6 @@ pub(crate) trait MapError: Sized { /// Maps errors to [`jsonrpsee_types::ErrorObjectOwned`] with a specific error code. fn map_error(self, code: impl Into) -> std::result::Result; - /// Maps errors to [`jsonrpsee_types::ErrorObjectOwned`] with a prefixed message and a specific error code. - fn map_error_with_prefix( - self, - code: impl Into, - msg_prefix: impl ToString, - ) -> Result; - /// Maps errors to [`jsonrpsee_types::ErrorObjectOwned`] with a [`LegacyCode::Misc`] error code. fn map_misc_error(self) -> std::result::Result { self.map_error(LegacyCode::Misc) @@ -105,20 +98,6 @@ where fn map_error(self, code: impl Into) -> Result { self.map_err(|error| ErrorObject::owned(code.into().code(), error.to_string(), None::<()>)) } - - fn map_error_with_prefix( - self, - code: impl Into, - msg_prefix: impl ToString, - ) -> Result { - self.map_err(|error| { - ErrorObject::owned( - code.into().code(), - format!("{}: {}", msg_prefix.to_string(), error.to_string()), - None::<()>, - ) - }) - } } impl OkOrError for Option { diff --git a/crates/zakura-rpc/tests/serialization_tests.rs b/crates/zakura-rpc/tests/serialization_tests.rs index 2d9b89bee0..0f4a14a7ea 100644 --- a/crates/zakura-rpc/tests/serialization_tests.rs +++ b/crates/zakura-rpc/tests/serialization_tests.rs @@ -1250,6 +1250,7 @@ fn test_get_block_template_response() -> Result<(), Box> let bits = template.bits().bytes_in_display_order(); let height = template.height(); let max_time = template.max_time(); + let work_id = template.work_id().clone(); let submit_old = template.submit_old(); let new_obj = GetBlockTemplateResponse::TemplateMode(Box::new(BlockTemplateResponse::new( @@ -1273,6 +1274,7 @@ fn test_get_block_template_response() -> Result<(), Box> CompactDifficulty::from_bytes_in_display_order(&bits).expect("was just serialized"), height, max_time, + work_id, submit_old, ))); diff --git a/crates/zakura-state/src/lib.rs b/crates/zakura-state/src/lib.rs index 0aae8a1d5e..27fe2018bb 100644 --- a/crates/zakura-state/src/lib.rs +++ b/crates/zakura-state/src/lib.rs @@ -56,7 +56,7 @@ pub use error::{ }; pub use header_chain::*; pub use request::{ - CheckpointVerifiedBlock, CommitSemanticallyVerifiedBlockRequest, HashOrHeight, + BlockAdmission, CheckpointVerifiedBlock, CommitSemanticallyVerifiedBlockRequest, HashOrHeight, HeaderChainBodyEvidenceAuthority, MappedRequest, PreparedHeaderChainBodyEvidence, PreparedHeaderChainInsert, ReadRequest, Request, SemanticallyVerifiedBlock, }; diff --git a/crates/zakura-state/src/request.rs b/crates/zakura-state/src/request.rs index 3ecfb4f855..f444ce60ac 100644 --- a/crates/zakura-state/src/request.rs +++ b/crates/zakura-state/src/request.rs @@ -5,11 +5,13 @@ use std::{ ops::{Add, Deref, RangeInclusive}, pin::Pin, sync::{ - atomic::{AtomicBool, Ordering}, + atomic::{AtomicBool, AtomicU8, Ordering}, Arc, }, }; +use tokio::sync::Notify; + use tower::{BoxError, Service, ServiceExt}; use zakura_chain::{ amount::{DeferredPoolBalanceChange, NegativeAllowed}, @@ -38,6 +40,82 @@ use crate::{ constants::{MAX_FIND_BLOCK_HASHES_RESULTS, MAX_FIND_BLOCK_HEADERS_RESULTS}, ReadResponse, Response, }; + +/// Notifies a mined-block submitter when state admits its block to the active write queue. +#[derive(Clone)] +pub struct BlockAdmission(Arc); + +#[derive(Debug)] +struct BlockAdmissionInner { + state: AtomicU8, + changed: Notify, +} + +impl BlockAdmission { + const PENDING: u8 = 0; + const ADMITTED: u8 = 1; + const REJECTED: u8 = 2; + + /// Creates a pending admission notification. + pub fn pending() -> Self { + Self(Arc::new(BlockAdmissionInner { + state: AtomicU8::new(Self::PENDING), + changed: Notify::new(), + })) + } + + /// Marks the block as admitted to the active non-finalized write queue. + pub(crate) fn admit(&self) { + self.0.state.store(Self::ADMITTED, Ordering::Release); + self.0.changed.notify_waiters(); + } + + /// Marks the block as rejected before admission. + pub(crate) fn reject(&self) { + if self + .0 + .state + .compare_exchange( + Self::PENDING, + Self::REJECTED, + Ordering::AcqRel, + Ordering::Acquire, + ) + .is_ok() + { + self.0.changed.notify_waiters(); + } + } + + /// Waits until state admits or rejects the block. + pub async fn wait(&self) -> bool { + loop { + let notified = self.0.changed.notified(); + match self.0.state.load(Ordering::Acquire) { + Self::ADMITTED => return true, + Self::REJECTED => return false, + Self::PENDING => notified.await, + _ => unreachable!("block admission state only uses declared constants"), + } + } + } +} + +impl std::fmt::Debug for BlockAdmission { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_tuple("BlockAdmission") + .field(&self.0.state.load(Ordering::Acquire)) + .finish() + } +} + +impl PartialEq for BlockAdmission { + fn eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.0, &other.0) + } +} + +impl Eq for BlockAdmission {} use crate::{ error::{CommitCheckpointVerifiedError, InvalidateError, LayeredStateError, ReconsiderError}, CommitSemanticallyVerifiedError, @@ -1105,6 +1183,14 @@ pub enum Request { /// [0]: (crate::error::CommitSemanticallyVerifiedError) CommitSemanticallyVerifiedBlock(SemanticallyVerifiedBlock), + /// Commits a mined block and reports when state admits it to the active write queue. + CommitSemanticallyVerifiedBlockWithAdmission { + /// The semantically verified mined block. + block: SemanticallyVerifiedBlock, + /// The admission notification. + admission: BlockAdmission, + }, + /// Commit a checkpointed block to the state, skipping most but not all /// contextual validation. /// @@ -1363,6 +1449,9 @@ impl Request { "retry_header_chain_body_availability" } Request::CommitSemanticallyVerifiedBlock(_) => "commit_semantically_verified_block", + Request::CommitSemanticallyVerifiedBlockWithAdmission { .. } => { + "commit_semantically_verified_block_with_admission" + } Request::CommitCheckpointVerifiedBlock(_) => "commit_checkpoint_verified_block", Request::AwaitUtxo(_) => "await_utxo", Request::Depth(_) => "depth", @@ -1992,6 +2081,7 @@ impl TryFrom for ReadRequest { | Request::RestartHeaderChainBodyAvailability { .. } | Request::RetryHeaderChainBodyAvailability { .. } | Request::CommitSemanticallyVerifiedBlock(_) + | Request::CommitSemanticallyVerifiedBlockWithAdmission { .. } | Request::CommitCheckpointVerifiedBlock(_) | Request::InvalidateBlock(_) | Request::ReconsiderBlock(_) => Err("ReadService does not write blocks"), diff --git a/crates/zakura-state/src/service.rs b/crates/zakura-state/src/service.rs index a50e2dc688..46c2e32faa 100644 --- a/crates/zakura-state/src/service.rs +++ b/crates/zakura-state/src/service.rs @@ -63,9 +63,9 @@ use crate::{ read::find, watch_receiver::WatchReceiver, }, - BoxError, CheckpointVerifiedBlock, CommitSemanticallyVerifiedError, Config, HashOrHeight, - HistoricalTreeUnavailable, KnownBlock, ReadRequest, ReadResponse, Request, Response, - SemanticallyVerifiedBlock, StateInitError, + BlockAdmission, BoxError, CheckpointVerifiedBlock, CommitSemanticallyVerifiedError, Config, + HashOrHeight, HistoricalTreeUnavailable, KnownBlock, ReadRequest, ReadResponse, Request, + Response, SemanticallyVerifiedBlock, StateInitError, }; pub mod block_iter; @@ -829,7 +829,11 @@ impl StateService { queued: QueuedSemanticallyVerified, error: impl Into, ) { - let (finalized, rsp_tx) = queued; + let (finalized, rsp_tx, admission) = queued; + + if let Some(admission) = admission { + admission.reject(); + } // The block sender might have already given up on this block, // so ignore any channel send errors. @@ -914,6 +918,7 @@ impl StateService { fn queue_and_commit_to_non_finalized_state( &mut self, semantically_verified: SemanticallyVerifiedBlock, + admission: Option, ) -> oneshot::Receiver> { tracing::debug!(block = %semantically_verified.block, "queueing block for contextual verification"); let parent_hash = semantically_verified.block.header.previous_block_hash; @@ -928,6 +933,9 @@ impl StateService { .non_finalized_block_write_sent_hashes .contains(&semantically_verified.hash) { + if let Some(admission) = admission { + admission.reject(); + } let (rsp_tx, rsp_rx) = oneshot::channel(); let _ = rsp_tx.send(Err(CommitBlockError::new_duplicate( Some(semantically_verified.hash.into()), @@ -942,6 +950,9 @@ impl StateService { .db .contains_height(semantically_verified.height) { + if let Some(admission) = admission { + admission.reject(); + } let (rsp_tx, rsp_rx) = oneshot::channel(); let _ = rsp_tx.send(Err(CommitBlockError::new_duplicate( Some(semantically_verified.height.into()), @@ -954,10 +965,13 @@ impl StateService { // [`Request::CommitSemanticallyVerifiedBlock`] contract: a request to commit a block which // has been queued but not yet committed to the state fails the older request and replaces // it with the newer request. - let rsp_rx = if let Some((_, old_rsp_tx)) = self + let rsp_rx = if let Some((_, old_rsp_tx, _)) = self .non_finalized_state_queued_blocks .get_mut(&semantically_verified.hash) { + if let Some(admission) = admission { + admission.reject(); + } tracing::debug!("replacing older queued request with new request"); let (mut rsp_tx, rsp_rx) = oneshot::channel(); std::mem::swap(old_rsp_tx, &mut rsp_tx); @@ -969,8 +983,11 @@ impl StateService { rsp_rx } else { let (rsp_tx, rsp_rx) = oneshot::channel(); - self.non_finalized_state_queued_blocks - .queue((semantically_verified, rsp_tx)); + self.non_finalized_state_queued_blocks.queue(( + semantically_verified, + rsp_tx, + admission, + )); rsp_rx }; @@ -1033,10 +1050,12 @@ impl StateService { .dequeue_children(parent_hash); for queued_child in queued_children { - let (SemanticallyVerifiedBlock { hash, .. }, _) = queued_child; + let (SemanticallyVerifiedBlock { hash, .. }, _, _) = &queued_child; + let hash = *hash; self.non_finalized_block_write_sent_hashes .add(&queued_child.0); + let admission = queued_child.2.clone(); let send_result = non_finalized_block_write_sender.send(queued_child.into()); if let Err(SendError(NonFinalizedWriteMessage::Commit(queued))) = send_result { @@ -1051,6 +1070,10 @@ impl StateService { return; }; + if let Some(admission) = admission { + admission.admit(); + } + new_parents.push(hash); } } @@ -1508,7 +1531,7 @@ impl Service for StateService { let rsp_rx = tokio::task::block_in_place(move || { span.in_scope(|| { - self.queue_and_commit_to_non_finalized_state(semantically_verified) + self.queue_and_commit_to_non_finalized_state(semantically_verified, None) }) }); @@ -1535,6 +1558,31 @@ impl Service for StateService { .boxed() } + Request::CommitSemanticallyVerifiedBlockWithAdmission { block, admission } => { + let timer = CodeTimer::start(); + self.assert_block_can_be_validated(&block); + self.pending_utxos.check_against_ordered(&block.new_outputs); + + let rsp_rx = tokio::task::block_in_place(move || { + span.in_scope(|| { + self.queue_and_commit_to_non_finalized_state(block, Some(admission)) + }) + }); + + timer.finish_desc("CommitSemanticallyVerifiedBlockWithAdmission"); + let span = Span::current(); + async move { + rsp_rx + .await + .map_err(|_recv_error| CommitBlockError::WriteTaskExited.into()) + .and_then(|result| result) + .map_err(BoxError::from) + .map(Response::Committed) + } + .instrument(span) + .boxed() + } + // Uses finalized_state_queued_blocks and pending_utxos in the StateService. // Accesses shared writeable state in the StateService. // diff --git a/crates/zakura-state/src/service/queued_blocks.rs b/crates/zakura-state/src/service/queued_blocks.rs index 3b4cb64d3d..749913cb85 100644 --- a/crates/zakura-state/src/service/queued_blocks.rs +++ b/crates/zakura-state/src/service/queued_blocks.rs @@ -12,8 +12,8 @@ use zakura_chain::{block, transparent}; use crate::{ error::{CommitBlockError, CommitCheckpointVerifiedError}, - CheckpointVerifiedBlock, CommitSemanticallyVerifiedError, KnownBlock, NonFinalizedState, - SemanticallyVerifiedBlock, + BlockAdmission, CheckpointVerifiedBlock, CommitSemanticallyVerifiedError, KnownBlock, + NonFinalizedState, SemanticallyVerifiedBlock, }; #[cfg(test)] @@ -29,6 +29,7 @@ pub type QueuedCheckpointVerified = ( pub type QueuedSemanticallyVerified = ( SemanticallyVerifiedBlock, oneshot::Sender>, + Option, ); /// A queue of blocks, awaiting the arrival of parent blocks. @@ -148,10 +149,14 @@ impl QueuedBlocks { mem::swap(&mut self.by_height, &mut by_height); for hash in by_height.into_values().flatten() { - let (expired_block, expired_sender) = + let (expired_block, expired_sender, admission) = self.blocks.remove(&hash).expect("block is present"); let parent_hash = &expired_block.block.header.previous_block_hash; + if let Some(admission) = admission { + admission.reject(); + } + // we don't care if the receiver was dropped let _ = expired_sender.send(Err(CommitBlockError::new_duplicate( Some(expired_block.height.into()), diff --git a/crates/zakura-state/src/service/queued_blocks/tests/vectors.rs b/crates/zakura-state/src/service/queued_blocks/tests/vectors.rs index b29c617bed..bff3b1ae0a 100644 --- a/crates/zakura-state/src/service/queued_blocks/tests/vectors.rs +++ b/crates/zakura-state/src/service/queued_blocks/tests/vectors.rs @@ -21,7 +21,7 @@ trait IntoQueued { impl IntoQueued for Arc { fn into_queued(self) -> QueuedSemanticallyVerified { let (rsp_tx, _) = oneshot::channel(); - (self.prepare(), rsp_tx) + (self.prepare(), rsp_tx, None) } } @@ -84,10 +84,10 @@ fn dequeue_gives_right_children() -> Result<()> { assert_eq!(2, children.len()); assert!(children .iter() - .any(|(block, _)| block.hash == child1.hash())); + .any(|(block, _, _)| block.hash == child1.hash())); assert!(children .iter() - .any(|(block, _)| block.hash == child2.hash())); + .any(|(block, _, _)| block.hash == child2.hash())); assert_eq!(0, queue.blocks.len()); assert_eq!(0, queue.by_parent.len()); assert_eq!(0, queue.by_height.len()); diff --git a/crates/zakura-state/src/service/tests.rs b/crates/zakura-state/src/service/tests.rs index c6e87fef9a..02f571d749 100644 --- a/crates/zakura-state/src/service/tests.rs +++ b/crates/zakura-state/src/service/tests.rs @@ -1157,7 +1157,8 @@ proptest! { let block_value_pool = &block.block.chain_value_pool_change(&transparent::utxos_from_ordered_utxos(utxos), None)?; expected_non_finalized_value_pool += *block_value_pool; - let result_receiver = state_service.queue_and_commit_to_non_finalized_state(block.clone()); + let result_receiver = + state_service.queue_and_commit_to_non_finalized_state(block.clone(), None); let result = result_receiver.blocking_recv(); prop_assert!(result.is_ok(), "unexpected failed non-finalized block commit: {:?}", result); @@ -1250,7 +1251,8 @@ proptest! { // every non-finalized block (height >= 1) grows the chain. let expected_action = TipAction::grow_with(expected_block.clone().into()); - let result_receiver = state_service.queue_and_commit_to_non_finalized_state(block); + let result_receiver = + state_service.queue_and_commit_to_non_finalized_state(block, None); let result = result_receiver.blocking_recv(); prop_assert!(result.is_ok(), "unexpected failed non-finalized block commit: {:?}", result); diff --git a/crates/zakura-state/src/service/write.rs b/crates/zakura-state/src/service/write.rs index c415b6fcfc..885e60c2af 100644 --- a/crates/zakura-state/src/service/write.rs +++ b/crates/zakura-state/src/service/write.rs @@ -2502,7 +2502,7 @@ impl WriteBlockWorkerTask { } }; - let Some((queued_child, rsp_tx)) = queued_child_and_rsp_tx else { + let Some((queued_child, rsp_tx, _admission)) = queued_child_and_rsp_tx else { continue; }; diff --git a/crates/zakurad/src/commands/start.rs b/crates/zakurad/src/commands/start.rs index ceead5d8c5..ab107bce9a 100644 --- a/crates/zakurad/src/commands/start.rs +++ b/crates/zakurad/src/commands/start.rs @@ -495,16 +495,18 @@ impl StartCmd { .then(|| config.state.pruning_config()) .flatten() .map(|pruning| pruning.tx_retention); + let pending_blocks = zakura_rpc::PendingBlockRegistry::default(); let inbound = ServiceBuilder::new() .load_shed() .buffer(inbound::downloads::MAX_INBOUND_CONCURRENCY) .timeout(MAX_INBOUND_RESPONSE_TIME) - .service(Inbound::new( + .service(Inbound::new_with_pending_blocks( config.sync.full_verify_concurrency_limit, config.network.expose_peer_addresses, zcashd_compat_pruning_retention, zcashd_compat_block_gossip_peer_ips.clone(), setup_rx, + pending_blocks.clone(), )); let advertised_services = Self::advertised_services(&config); @@ -638,7 +640,7 @@ impl StartCmd { let submit_block_channel = SubmitBlockChannel::new(); // Launch RPC server - let (rpc_impl, mut rpc_tx_queue_handle) = RpcImpl::new( + let (rpc_impl, mut rpc_tx_queue_handle) = RpcImpl::new_with_pending_blocks( config.network.network.clone(), config.mining.clone(), config.rpc.debug_force_finished_sync, @@ -653,6 +655,7 @@ impl StartCmd { address_book.clone(), LAST_WARN_ERROR_LOG_SENDER.subscribe(), Some(submit_block_channel.sender()), + pending_blocks, ); node_tasks.track(&rpc_tx_queue_handle); let rpc_impl = rpc_impl.with_end_of_support_height( diff --git a/crates/zakurad/src/components/inbound.rs b/crates/zakurad/src/components/inbound.rs index 34c0cf14fa..cf8527dc32 100644 --- a/crates/zakurad/src/components/inbound.rs +++ b/crates/zakurad/src/components/inbound.rs @@ -17,7 +17,7 @@ use std::{ use futures::{ future::{FutureExt, TryFutureExt}, - stream::Stream, + stream::{FuturesUnordered, Stream, StreamExt}, }; use tokio::sync::oneshot::{self, error::TryRecvError}; use tower::{buffer::Buffer, timeout::Timeout, util::BoxService, Service, ServiceExt}; @@ -33,6 +33,7 @@ use zakura_chain::{ use zakura_consensus::{router::RouterError, VerifyBlockError}; use zakura_network::{AddressBook, InventoryResponse}; use zakura_node_services::mempool; +use zakura_rpc::PendingBlockRegistry; use crate::BoxError; @@ -55,7 +56,7 @@ use downloads::{Downloads as BlockDownloads, GossipedTipChildHeightMismatch}; /// /// If the response takes longer than this time, it will be cancelled, /// and the peer might be disconnected. -pub const MAX_INBOUND_RESPONSE_TIME: Duration = Duration::from_secs(5); +pub const MAX_INBOUND_RESPONSE_TIME: Duration = Duration::from_secs(18); /// The number of bytes the [`Inbound`] service will queue in response to a single block or /// transaction request, before ignoring any additional block or transaction IDs in that request. @@ -338,6 +339,9 @@ pub struct Inbound { /// Diagnostics for zcashd-compat requests that need pruned block bodies. pruned_block_not_found_logger: Arc, + + /// Early-advertised mined blocks waiting for contextual commit. + pending_blocks: PendingBlockRegistry, } impl Inbound { @@ -350,6 +354,25 @@ impl Inbound { zcashd_compat_pruning_retention: Option, zcashd_compat_peer_ips: Vec, setup: oneshot::Receiver, + ) -> Inbound { + Self::new_with_pending_blocks( + full_verify_concurrency_limit, + expose_peer_addresses, + zcashd_compat_pruning_retention, + zcashd_compat_peer_ips, + setup, + PendingBlockRegistry::default(), + ) + } + + /// Creates an inbound service with a pending-block registry shared with mining RPCs. + pub fn new_with_pending_blocks( + full_verify_concurrency_limit: usize, + expose_peer_addresses: bool, + zcashd_compat_pruning_retention: Option, + zcashd_compat_peer_ips: Vec, + setup: oneshot::Receiver, + pending_blocks: PendingBlockRegistry, ) -> Inbound { Inbound { setup: Setup::Pending { @@ -361,6 +384,7 @@ impl Inbound { zcashd_compat_pruning_retention, zcashd_compat_peer_ips, )), + pending_blocks, } } @@ -536,6 +560,7 @@ impl Service for Inbound { #[instrument(name = "inbound", skip(self, req))] fn call(&mut self, req: zn::Request) -> Self::Future { let pruned_block_not_found_logger = self.pruned_block_not_found_logger.clone(); + let pending_blocks = self.pending_blocks.clone(); let (cached_peer_addr_response, block_downloads, mempool, state) = match &mut self.setup { Setup::Initialized { cached_peer_addr_response, @@ -592,24 +617,48 @@ impl Service for Inbound { async move { let mut blocks: Vec, Option), block::Hash>> = Vec::new(); let mut total_size = 0; + let mut lookups = FuturesUnordered::new(); + + for (index, &hash) in hashes.iter().take(GETDATA_MAX_BLOCK_COUNT).enumerate() { + let state = state.clone(); + let pending_blocks = pending_blocks.clone(); + lookups.push(async move { + let response = state + .clone() + .ready() + .await? + .call(zs::Request::Block(hash.into())) + .await?; + match response { + zs::Response::Block(Some(block)) => { + Ok::<_, zn::BoxError>((index, hash, Some(block))) + } + zs::Response::Block(None) => { + Ok((index, hash, pending_blocks.wait(hash).await)) + } + _ => unreachable!("wrong response from state"), + } + }); + } - // Ignore any block hashes past the response limit. - // This saves us expensive database lookups. - for &hash in hashes.iter().take(GETDATA_MAX_BLOCK_COUNT) { - // We check the limit after including at least one block, so that we can - // send blocks greater than 1 MB (but only one at a time) + // Start every state lookup and pending wait before awaiting any result. + let mut lookup_results = Vec::with_capacity(lookups.len()); + while let Some(result) = lookups.next().await { + lookup_results.push(result?); + } + lookup_results.sort_unstable_by_key(|(index, _, _)| *index); + + for (_, hash, block) in lookup_results { if total_size >= GETDATA_SENT_BYTES_LIMIT { - break; + continue; } - let response = state.clone().ready().await?.call(zs::Request::Block(hash.into())).await?; - // Add the block responses to the list, while updating the size limit. // // If there was a database error, return the error, // and stop processing further chunks. - match response { - zs::Response::Block(Some(block)) => { + match block { + Some(block) => { // If checking the serialized size of the block performs badly, // return the size from the state using a wrapper type. total_size += block.zcash_serialized_size(); @@ -619,7 +668,7 @@ impl Service for Inbound { // We don't need to limit the size of the missing block IDs list, // because it is already limited to the size of the getdata request // sent by the peer. (Their content and encodings are the same.) - zs::Response::Block(None) => { + None => { // A retained canonical header with no block body identifies // history removed by pruning. Unknown hashes remain ordinary // `notfound` responses without reserving a log interval. @@ -642,9 +691,7 @@ impl Service for Inbound { blocks.push(Missing(hash)) }, - _ => unreachable!("wrong response from state"), } - } // The network layer handles splitting this response into multiple `block` diff --git a/crates/zakurad/src/components/inbound/tests/real_peer_set.rs b/crates/zakurad/src/components/inbound/tests/real_peer_set.rs index c0e324b43d..96038dfae7 100644 --- a/crates/zakurad/src/components/inbound/tests/real_peer_set.rs +++ b/crates/zakurad/src/components/inbound/tests/real_peer_set.rs @@ -1129,7 +1129,7 @@ async fn setup( mod submitblock_test { use tracing::{Instrument, Level}; use tracing_subscriber::fmt; - use zakura_rpc::SubmitBlockChannel; + use zakura_rpc::{MinedBlockEvent, SubmitBlockChannel}; use super::*; @@ -1200,7 +1200,11 @@ mod submitblock_test { // Send a block to the channel submitblock_channel .sender() - .send((block::Hash([1; 32]), block::Height(1))) + .send(MinedBlockEvent::Committed { + hash: block::Hash([1; 32]), + height: block::Height(1), + early_advertised: false, + }) .await .unwrap(); let gossip_task_handle = tokio::spawn( @@ -1221,7 +1225,7 @@ mod submitblock_test { let sent_mined_block = { let captured_logs = logs.lock().unwrap(); String::from_utf8_lossy(&captured_logs) - .contains("sending mined block broadcast") + .contains("sending committed mined block broadcast") }; if sent_mined_block { @@ -1241,7 +1245,7 @@ mod submitblock_test { }; assert!(log_output.contains("initializing block gossip task")); - assert!(log_output.contains("sending mined block broadcast")); + assert!(log_output.contains("sending committed mined block broadcast")); gossip_task_handle.abort(); let gossip_task_error = gossip_task_handle diff --git a/crates/zakurad/src/components/sync/gossip.rs b/crates/zakurad/src/components/sync/gossip.rs index a059727422..7f678b5e87 100644 --- a/crates/zakurad/src/components/sync/gossip.rs +++ b/crates/zakurad/src/components/sync/gossip.rs @@ -12,6 +12,7 @@ use tracing::Instrument; use zakura_chain::block; use zakura_network as zn; +use zakura_rpc::MinedBlockEvent; use zakura_state::ChainTipChange; use crate::{ @@ -29,15 +30,15 @@ use BlockGossipError::*; /// is chosen arbitrarily high to be safe. const MINED_BLOCK_MARK_CHANNEL_CAPACITY: usize = 16; -#[derive(Debug, Eq, PartialEq)] +#[derive(Debug)] enum GossipEvent { MinedBlockBroadcastCompleted(block::Hash), - MinedBlockSubmitted((block::Hash, block::Height)), + MinedBlock(MinedBlockEvent), CommittedTip(T), } async fn next_gossip_event( - mined_block_receiver: Option<&mut mpsc::Receiver<(block::Hash, block::Height)>>, + mined_block_receiver: Option<&mut mpsc::Receiver>, mined_block_mark_receiver: &mut mpsc::Receiver, committed_tip_fut: impl Future, ) -> GossipEvent { @@ -50,7 +51,7 @@ async fn next_gossip_event( }, Some(tip_change) = mined_block_receiver.recv() => { - GossipEvent::MinedBlockSubmitted(tip_change) + GossipEvent::MinedBlock(tip_change) }, committed_tip = committed_tip_fut => { @@ -102,7 +103,7 @@ pub async fn gossip_best_tip_block_hashes( sync_status: SyncStatus, mut chain_state: ChainTipChange, broadcast_network: ZN, - mut mined_block_receiver: Option>, + mut mined_block_receiver: Option>, ) -> Result<(), BlockGossipError> where ZN: Service + Send + Clone + 'static, @@ -177,7 +178,7 @@ where // Prefer mined-block completions and submissions when multiple // branches are ready. The committed-tip path is a fallback, so // selecting it first can duplicate a mined-block broadcast. - let (((hash, height), log_msg, updated_chain_state), is_block_submission) = + let (((hash, height), log_msg, updated_chain_state), is_block_submission, early_ack) = match next_gossip_event( mined_block_receiver.as_mut(), &mut mined_block_mark_receiver, @@ -189,12 +190,59 @@ where chain_state.mark_last_change_hash(mark_hash); continue; } - GossipEvent::MinedBlockSubmitted(tip_change) => ( - (tip_change, "sending mined block broadcast", chain_state), + GossipEvent::MinedBlock(MinedBlockEvent::Early { + hash, + height, + submitted_at, + advertised, + }) => ( + ( + (hash, height), + "sending early mined block broadcast", + chain_state, + ), true, + Some((advertised, submitted_at)), ), + GossipEvent::MinedBlock(MinedBlockEvent::Committed { + hash, + height: _, + early_advertised: true, + }) => { + chain_state.mark_last_change_hash(hash); + continue; + } + GossipEvent::MinedBlock(MinedBlockEvent::Committed { + hash, + height, + early_advertised: false, + }) => { + metrics::counter!("mining.optimistic_inventory.fallbacks").increment(1); + ( + ( + (hash, height), + "sending committed mined block broadcast", + chain_state, + ), + true, + None, + ) + } + GossipEvent::MinedBlock(MinedBlockEvent::Failed { + hash, + height, + early_advertised, + }) => { + tracing::debug!( + ?hash, + ?height, + early_advertised, + "mined block lifecycle failed" + ); + continue; + } GossipEvent::CommittedTip(tip_change_close_to_network_tip) => { - (tip_change_close_to_network_tip?, false) + (tip_change_close_to_network_tip?, false, None) } }; @@ -224,9 +272,19 @@ where let mark_tx = mined_block_mark_sender.clone(); let submission_hash = hash; tokio::spawn(async move { - if broadcast_fut.await.is_ok() { + let succeeded = broadcast_fut.await.is_ok(); + if succeeded { let _ = mark_tx.send(submission_hash).await; } + if let Some((advertised, submitted_at)) = early_ack { + if succeeded { + metrics::counter!("mining.optimistic_inventory.early_inventories") + .increment(1); + metrics::histogram!("mining.submit_to_inventory.duration_seconds") + .record(submitted_at.elapsed().as_secs_f64()); + } + let _ = advertised.send(succeeded); + } }); } else { tokio::spawn(broadcast_fut); @@ -242,6 +300,7 @@ mod tests { use tokio::sync::mpsc; use zakura_chain::block; + use zakura_rpc::MinedBlockEvent; // Repeat the vector so removing `biased;` reliably exposes randomized // selection among the ready events. @@ -249,14 +308,21 @@ mod tests { #[tokio::test] async fn ready_gossip_events_are_selected_in_priority_order() { - let submitted_block = (block::Hash([1; 32]), block::Height(1)); + let submitted_hash = block::Hash([1; 32]); for _ in 0..READY_EVENT_ATTEMPTS { let (mined_block_sender, mut mined_block_receiver) = mpsc::channel(1); let (mark_sender, mut mark_receiver) = mpsc::channel(1); - mined_block_sender.send(submitted_block).await.unwrap(); - mark_sender.send(submitted_block.0).await.unwrap(); + mined_block_sender + .send(MinedBlockEvent::Committed { + hash: submitted_hash, + height: block::Height(1), + early_advertised: false, + }) + .await + .unwrap(); + mark_sender.send(submitted_hash).await.unwrap(); let event = next_gossip_event( Some(&mut mined_block_receiver), @@ -265,10 +331,10 @@ mod tests { ) .await; - assert_eq!( + assert!(matches!( event, - GossipEvent::MinedBlockBroadcastCompleted(submitted_block.0) - ); + GossipEvent::MinedBlockBroadcastCompleted(hash) if hash == submitted_hash + )); let event = next_gossip_event( Some(&mut mined_block_receiver), @@ -277,7 +343,14 @@ mod tests { ) .await; - assert_eq!(event, GossipEvent::MinedBlockSubmitted(submitted_block)); + assert!(matches!( + event, + GossipEvent::MinedBlock(MinedBlockEvent::Committed { + hash, + height: block::Height(1), + early_advertised: false, + }) if hash == submitted_hash + )); let event = next_gossip_event( Some(&mut mined_block_receiver), @@ -286,7 +359,7 @@ mod tests { ) .await; - assert_eq!(event, GossipEvent::CommittedTip("committed tip")); + assert!(matches!(event, GossipEvent::CommittedTip("committed tip"))); } } } diff --git a/crates/zakurad/src/components/sync/tests/gossip.rs b/crates/zakurad/src/components/sync/tests/gossip.rs index e0316928a9..de55aef214 100644 --- a/crates/zakurad/src/components/sync/tests/gossip.rs +++ b/crates/zakurad/src/components/sync/tests/gossip.rs @@ -15,7 +15,7 @@ use zakura_chain::{ serialization::ZcashDeserializeInto, }; use zakura_network::{Request, Response}; -use zakura_rpc::SubmitBlockChannel; +use zakura_rpc::{MinedBlockEvent, SubmitBlockChannel}; use zakura_state::{Config as StateConfig, CHAIN_TIP_UPDATE_WAIT_LIMIT}; use zakura_test::mock_service::{MockService, PanicAssertion}; @@ -27,7 +27,7 @@ const MAX_PEER_SET_REQUEST_DELAY: Duration = Duration::from_secs(30); struct GossipTestSetup { peer_set: MockService, - submitblock_sender: tokio::sync::mpsc::Sender<(zakura_chain::block::Hash, Height)>, + submitblock_sender: tokio::sync::mpsc::Sender, state_service: BoxService, gossip_task_handle: JoinHandle>, } @@ -141,7 +141,11 @@ async fn mined_block_marks_tip_after_successful_broadcast() { .unwrap(); submitblock_sender - .send((block_two.hash(), block_two.coinbase_height().unwrap())) + .send(MinedBlockEvent::Committed { + hash: block_two.hash(), + height: block_two.coinbase_height().unwrap(), + early_advertised: false, + }) .await .expect("mined block notification should be accepted"); @@ -187,7 +191,11 @@ async fn mined_block_mark_survives_pending_submit_queue() { // First mined notification — start AdvertiseBlockToAll but hold the response open. submitblock_sender - .send((hash, height)) + .send(MinedBlockEvent::Committed { + hash, + height, + early_advertised: false, + }) .await .expect("mined block notification should be accepted"); @@ -198,7 +206,11 @@ async fn mined_block_mark_survives_pending_submit_queue() { // Queue a second notification while the first broadcast is still in flight so the // submit-block channel is nonempty when the first mark arrives. submitblock_sender - .send((hash, height)) + .send(MinedBlockEvent::Committed { + hash, + height, + early_advertised: false, + }) .await .expect("second mined block notification should be accepted"); @@ -245,7 +257,11 @@ async fn mined_block_broadcast_timeout_uses_committed_tip_fallback() { .unwrap(); submitblock_sender - .send((block_two.hash(), block_two.coinbase_height().unwrap())) + .send(MinedBlockEvent::Committed { + hash: block_two.hash(), + height: block_two.coinbase_height().unwrap(), + early_advertised: false, + }) .await .expect("mined block notification should be accepted"); diff --git a/crates/zakurad/tests/acceptance.rs b/crates/zakurad/tests/acceptance.rs index eb57eae632..adc7db827f 100644 --- a/crates/zakurad/tests/acceptance.rs +++ b/crates/zakurad/tests/acceptance.rs @@ -177,7 +177,7 @@ use zakura_rpc::{ methods::{RpcImpl, RpcServer}, proposal_block_from_template, server::OPENED_RPC_ENDPOINT_MSG, - MinerParams, SubmitBlockChannel, + MinedBlockEvent, MinerParams, SubmitBlockChannel, }; use zakura_state::{constants::LOCK_FILE_ERROR, state_database_format_version_in_code}; use zakura_test::{ @@ -3822,13 +3822,14 @@ async fn nu6_funding_streams_and_coinbase_balance() -> Result<()> { // Check that the submitblock channel received the submitted block let mut submit_block_receiver = submitblock_channel.receiver(); let submit_block_channel_data = submit_block_receiver.recv().await.expect("channel is open"); - assert_eq!( - submit_block_channel_data, - ( - proposal_block.hash(), - proposal_block.coinbase_height().unwrap() + assert!( + matches!( + submit_block_channel_data, + MinedBlockEvent::Early { hash, height, .. } + if hash == proposal_block.hash() + && height == proposal_block.coinbase_height().unwrap() ), - "submitblock channel should receive the submitted block" + "submitblock channel should receive the early submitted-block event" ); // Use an invalid coinbase transaction (with an output value greater than the `block_subsidy + miner_fees - expected_lockbox_funding_stream`) From f27cb7f77acd25a19995a4ee39f8829b039a4e95 Mon Sep 17 00:00:00 2001 From: evan-forbes Date: Thu, 20 Aug 2026 09:14:03 -0500 Subject: [PATCH 02/22] fix(mining): harden optimistic inventory --- crates/zakura-consensus/src/block/prepared.rs | 22 ++++--- crates/zakura-rpc/src/methods.rs | 1 - .../src/methods/types/submit_block.rs | 63 ++++++++++++------- crates/zakurad/src/components/inbound.rs | 5 +- crates/zakurad/tests/acceptance.rs | 2 + docs/changelog/unreleased/748.md | 4 ++ 6 files changed, 67 insertions(+), 30 deletions(-) create mode 100644 docs/changelog/unreleased/748.md diff --git a/crates/zakura-consensus/src/block/prepared.rs b/crates/zakura-consensus/src/block/prepared.rs index b0a8b899d8..24d8e3747f 100644 --- a/crates/zakura-consensus/src/block/prepared.rs +++ b/crates/zakura-consensus/src/block/prepared.rs @@ -100,18 +100,13 @@ impl PreparedCandidateCache { ) { let immutable_bytes = immutable_candidate_bytes(block, network); let fingerprint = fingerprint(&immutable_bytes); - // Count the canonical candidate and the derived verification inputs conservatively. - let size = immutable_bytes.len().saturating_mul(2); - if size > MAX_BYTES { - return; - } let mut inner = self .0 .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); inner.prune_expired(); - let work_id = work_id.map(ToOwned::to_owned).or_else(|| { + let existing_work_id = if work_id.is_none() { inner .entries .iter() @@ -119,7 +114,20 @@ impl PreparedCandidateCache { entry.fingerprint == fingerprint && entry.immutable_bytes == immutable_bytes }) .and_then(|entry| entry.work_id.clone()) - }); + } else { + None + }; + // Count the canonical candidate, derived verification inputs, and caller-supplied work ID. + let size = immutable_bytes.len().saturating_mul(2).saturating_add( + work_id + .map(str::len) + .or_else(|| existing_work_id.as_ref().map(String::len)) + .unwrap_or(0), + ); + if size > MAX_BYTES { + return; + } + let work_id = work_id.map(ToOwned::to_owned).or(existing_work_id); inner.remove_matching(fingerprint, &immutable_bytes); while inner.entries.len() >= MAX_ENTRIES || inner.bytes.saturating_add(size) > MAX_BYTES { diff --git a/crates/zakura-rpc/src/methods.rs b/crates/zakura-rpc/src/methods.rs index 006159c80f..e9c781f7be 100644 --- a/crates/zakura-rpc/src/methods.rs +++ b/crates/zakura-rpc/src/methods.rs @@ -1047,7 +1047,6 @@ where #[cfg(test)] { let _ = template; - return; } #[cfg(not(test))] diff --git a/crates/zakura-rpc/src/methods/types/submit_block.rs b/crates/zakura-rpc/src/methods/types/submit_block.rs index 9ea040ac5a..7ecee50621 100644 --- a/crates/zakura-rpc/src/methods/types/submit_block.rs +++ b/crates/zakura-rpc/src/methods/types/submit_block.rs @@ -131,34 +131,42 @@ impl PendingBlockRegistry { } /// Waits for an early-advertised block to commit. - pub async fn wait(&self, hash: block::Hash) -> Option> { - let mut status = self + pub fn wait( + &self, + hash: block::Hash, + ) -> impl std::future::Future>> + Send + 'static { + let status = self .0 .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .get(&hash) - .map(|entry| entry.status.subscribe())?; - let start = std::time::Instant::now(); - let result = tokio::time::timeout(PENDING_BLOCK_WAIT, async { - loop { - let current = status.borrow().clone(); - match current { - PendingStatus::Waiting => { - if status.changed().await.is_err() { - return None; + .map(|entry| entry.status.subscribe()); + let deadline = tokio::time::Instant::now() + PENDING_BLOCK_WAIT; + + async move { + let mut status = status?; + let start = std::time::Instant::now(); + let result = tokio::time::timeout_at(deadline, async { + loop { + let current = status.borrow().clone(); + match current { + PendingStatus::Waiting => { + if status.changed().await.is_err() { + return None; + } } + PendingStatus::Committed(block) => return Some(block), + PendingStatus::Failed => return None, } - PendingStatus::Committed(block) => return Some(block), - PendingStatus::Failed => return None, } - } - }) - .await - .ok() - .flatten(); - metrics::histogram!("mining.pending_peer_wait.duration_seconds") - .record(start.elapsed().as_secs_f64()); - result + }) + .await + .ok() + .flatten(); + metrics::histogram!("mining.pending_peer_wait.duration_seconds") + .record(start.elapsed().as_secs_f64()); + result + } } } @@ -270,6 +278,19 @@ mod tests { assert_eq!(wait.await.expect("wait task succeeds"), Some(block)); } + #[tokio::test] + async fn pending_block_wait_subscribes_before_polling() { + let registry = PendingBlockRegistry::default(); + let block = test_block(); + let hash = block.hash(); + assert!(registry.insert(block.clone())); + + let wait = registry.wait(hash); + registry.resolve(hash, Ok(block.clone())); + + assert_eq!(wait.await, Some(block)); + } + #[tokio::test] async fn pending_block_failure_returns_not_found() { let registry = PendingBlockRegistry::default(); diff --git a/crates/zakurad/src/components/inbound.rs b/crates/zakurad/src/components/inbound.rs index cf8527dc32..dbcdbd58a2 100644 --- a/crates/zakurad/src/components/inbound.rs +++ b/crates/zakurad/src/components/inbound.rs @@ -623,6 +623,9 @@ impl Service for Inbound { let state = state.clone(); let pending_blocks = pending_blocks.clone(); lookups.push(async move { + // Subscribe before the state lookup. A commit can complete and + // remove the registry entry while state answers this request. + let pending_wait = pending_blocks.wait(hash); let response = state .clone() .ready() @@ -634,7 +637,7 @@ impl Service for Inbound { Ok::<_, zn::BoxError>((index, hash, Some(block))) } zs::Response::Block(None) => { - Ok((index, hash, pending_blocks.wait(hash).await)) + Ok((index, hash, pending_wait.await)) } _ => unreachable!("wrong response from state"), } diff --git a/crates/zakurad/tests/acceptance.rs b/crates/zakurad/tests/acceptance.rs index adc7db827f..4eba4ea774 100644 --- a/crates/zakurad/tests/acceptance.rs +++ b/crates/zakurad/tests/acceptance.rs @@ -3909,6 +3909,7 @@ async fn nu6_funding_streams_and_coinbase_balance() -> Result<()> { block_template.bits(), block_template.height(), block_template.max_time(), + block_template.work_id().clone(), block_template.submit_old(), ); @@ -3974,6 +3975,7 @@ async fn nu6_funding_streams_and_coinbase_balance() -> Result<()> { block_template.bits(), block_template.height(), block_template.max_time(), + block_template.work_id().clone(), block_template.submit_old(), ); diff --git a/docs/changelog/unreleased/748.md b/docs/changelog/unreleased/748.md new file mode 100644 index 0000000000..3555c645d7 --- /dev/null +++ b/docs/changelog/unreleased/748.md @@ -0,0 +1,4 @@ +## Added + +- Added early mined-block inventory after state admission. +- Added prepared mining-candidate reuse through `workid`. From 5a6a7a9cbc58383e248d6c3a31ed2c7e2fba4e17 Mon Sep 17 00:00:00 2001 From: evan-forbes Date: Thu, 20 Aug 2026 12:33:45 -0500 Subject: [PATCH 03/22] fix(mining): correct optimistic inventory validation --- crates/zakura-consensus/src/block.rs | 21 ++++++-- crates/zakura-consensus/src/block/prepared.rs | 48 +++++++++++++++++-- crates/zakura-rpc/src/methods.rs | 4 +- crates/zakura-state/src/request.rs | 28 ++++++++++- crates/zakurad/src/components/sync/gossip.rs | 19 ++++---- .../tests/common/configs/v1.3.0-rc1.toml | 1 + 6 files changed, 99 insertions(+), 22 deletions(-) diff --git a/crates/zakura-consensus/src/block.rs b/crates/zakura-consensus/src/block.rs index 47efd06e52..9667c6972d 100644 --- a/crates/zakura-consensus/src/block.rs +++ b/crates/zakura-consensus/src/block.rs @@ -305,8 +305,18 @@ where if let Some(mut prepared_block) = prepared_candidates.lookup(&block, request.work_id(), &network) { - check::difficulty_is_valid(&block.header, &network, &height, &hash)?; - check::equihash_solution_is_valid(&block.header, &network)?; + let pow_policy = zakura_header_chain::PowPolicy::for_network(&network)?; + if pow_policy.is_authenticated_custom_waiver() { + check::difficulty_threshold_is_valid( + &block.header, + &network, + &height, + &hash, + )?; + } else { + check::difficulty_is_valid(&block.header, &network, &height, &hash)?; + check::equihash_solution_is_valid(&block.header, &network)?; + } check::time_is_valid_at(&block.header, Utc::now(), &height, &hash) .map_err(VerifyBlockError::Time)?; for transaction in &block.transactions { @@ -520,6 +530,7 @@ where S::Future: Send + 'static, { let hash = prepared_block.hash; + let is_mined_commit = admission.is_some(); let request = match admission { Some(admission) => zs::Request::CommitSemanticallyVerifiedBlockWithAdmission { block: prepared_block, @@ -534,8 +545,10 @@ where .map_err(|source| VerifyBlockError::StateService { source, hash })? .call(request) .await; - metrics::histogram!("mining.contextual_commit.duration_seconds") - .record(commit_start.elapsed().as_secs_f64()); + if is_mined_commit { + metrics::histogram!("mining.contextual_commit.duration_seconds") + .record(commit_start.elapsed().as_secs_f64()); + } match response { Ok(zs::Response::Committed(committed_hash)) => { diff --git a/crates/zakura-consensus/src/block/prepared.rs b/crates/zakura-consensus/src/block/prepared.rs index 24d8e3747f..371ca2c0db 100644 --- a/crates/zakura-consensus/src/block/prepared.rs +++ b/crates/zakura-consensus/src/block/prepared.rs @@ -128,7 +128,7 @@ impl PreparedCandidateCache { return; } let work_id = work_id.map(ToOwned::to_owned).or(existing_work_id); - inner.remove_matching(fingerprint, &immutable_bytes); + inner.remove_matching(work_id.as_deref(), fingerprint, &immutable_bytes); while inner.entries.len() >= MAX_ENTRIES || inner.bytes.saturating_add(size) > MAX_BYTES { if !inner.evict_oldest() { @@ -160,9 +160,15 @@ impl CacheInner { } } - fn remove_matching(&mut self, fingerprint: [u8; 32], immutable_bytes: &[u8]) { - if let Some(index) = self.entries.iter().position(|entry| { - entry.fingerprint == fingerprint && entry.immutable_bytes == immutable_bytes + fn remove_matching( + &mut self, + work_id: Option<&str>, + fingerprint: [u8; 32], + immutable_bytes: &[u8], + ) { + while let Some(index) = self.entries.iter().position(|entry| { + work_id.is_some_and(|work_id| entry.work_id.as_deref() == Some(work_id)) + || (entry.fingerprint == fingerprint && entry.immutable_bytes == immutable_bytes) }) { let entry = self .entries @@ -275,4 +281,38 @@ mod tests { .lookup(&changed_transactions, Some("work"), &network) .is_none()); } + + #[test] + fn inserting_a_reused_work_id_replaces_the_old_candidate() { + let network = Network::Mainnet; + let original = test_block(); + let mut replacement = original.clone(); + Arc::make_mut(&mut replacement.header).version ^= 1; + let cache = PreparedCandidateCache::default(); + + cache.insert( + &original, + Some("work"), + SemanticallyVerifiedBlock::from(Arc::new(original.clone())), + &network, + ); + cache.insert( + &replacement, + Some("work"), + SemanticallyVerifiedBlock::from(Arc::new(replacement.clone())), + &network, + ); + + assert!(cache.lookup(&replacement, Some("work"), &network).is_some()); + assert!(cache.lookup(&original, Some("work"), &network).is_none()); + assert_eq!( + cache + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .entries + .len(), + 1 + ); + } } diff --git a/crates/zakura-rpc/src/methods.rs b/crates/zakura-rpc/src/methods.rs index e9c781f7be..781646f9fb 100644 --- a/crates/zakura-rpc/src/methods.rs +++ b/crates/zakura-rpc/src/methods.rs @@ -2839,7 +2839,6 @@ where if mined_block_sender.try_send(event).is_ok() { early_result = Some(receiver); } else { - metrics::counter!("mining.optimistic_inventory.fallbacks").increment(1); pending_blocks.resolve(block_hash, Err(())); } } @@ -2867,6 +2866,9 @@ where None => false, }; let event = if committed { + if optimistic_block_inventory && !early_advertised { + metrics::counter!("mining.optimistic_inventory.fallbacks").increment(1); + } MinedBlockEvent::Committed { hash: block_hash, height, diff --git a/crates/zakura-state/src/request.rs b/crates/zakura-state/src/request.rs index f444ce60ac..21e72ce4c6 100644 --- a/crates/zakura-state/src/request.rs +++ b/crates/zakura-state/src/request.rs @@ -66,8 +66,19 @@ impl BlockAdmission { /// Marks the block as admitted to the active non-finalized write queue. pub(crate) fn admit(&self) { - self.0.state.store(Self::ADMITTED, Ordering::Release); - self.0.changed.notify_waiters(); + if self + .0 + .state + .compare_exchange( + Self::PENDING, + Self::ADMITTED, + Ordering::AcqRel, + Ordering::Acquire, + ) + .is_ok() + { + self.0.changed.notify_waiters(); + } } /// Marks the block as rejected before admission. @@ -788,6 +799,19 @@ mod tests { assert_eq!(checkpoint.auth_data_root, Some(block.auth_data_root())); } + + #[tokio::test] + async fn block_admission_keeps_its_first_terminal_state() { + let rejected = BlockAdmission::pending(); + rejected.reject(); + rejected.admit(); + assert!(!rejected.wait().await); + + let admitted = BlockAdmission::pending(); + admitted.admit(); + admitted.reject(); + assert!(admitted.wait().await); + } } impl From for SemanticallyVerifiedBlock { diff --git a/crates/zakurad/src/components/sync/gossip.rs b/crates/zakurad/src/components/sync/gossip.rs index 7f678b5e87..65ab336141 100644 --- a/crates/zakurad/src/components/sync/gossip.rs +++ b/crates/zakurad/src/components/sync/gossip.rs @@ -216,18 +216,15 @@ where hash, height, early_advertised: false, - }) => { - metrics::counter!("mining.optimistic_inventory.fallbacks").increment(1); + }) => ( ( - ( - (hash, height), - "sending committed mined block broadcast", - chain_state, - ), - true, - None, - ) - } + (hash, height), + "sending committed mined block broadcast", + chain_state, + ), + true, + None, + ), GossipEvent::MinedBlock(MinedBlockEvent::Failed { hash, height, diff --git a/crates/zakurad/tests/common/configs/v1.3.0-rc1.toml b/crates/zakurad/tests/common/configs/v1.3.0-rc1.toml index f4354de84f..2a6a188d1f 100644 --- a/crates/zakurad/tests/common/configs/v1.3.0-rc1.toml +++ b/crates/zakurad/tests/common/configs/v1.3.0-rc1.toml @@ -61,6 +61,7 @@ tx_cost_limit = 80000000 [mining] internal_miner = false +optimistic_block_inventory = true [network] cache_dir = true From 2b3b8fa56144a4cf563f21d13cbafa95731bf169 Mon Sep 17 00:00:00 2001 From: evan-forbes Date: Thu, 20 Aug 2026 13:06:40 -0500 Subject: [PATCH 04/22] test(rpc): include workid in template fixture --- .../tests/vectors/getblocktemplate_response_template.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/zakura-rpc/tests/vectors/getblocktemplate_response_template.json b/crates/zakura-rpc/tests/vectors/getblocktemplate_response_template.json index 78568bf56c..a5d97920dc 100644 --- a/crates/zakura-rpc/tests/vectors/getblocktemplate_response_template.json +++ b/crates/zakura-rpc/tests/vectors/getblocktemplate_response_template.json @@ -37,5 +37,6 @@ "curtime": 1747848159, "bits": "1c01da67", "height": 2931867, - "maxtime": 1747853232 -} \ No newline at end of file + "maxtime": 1747853232, + "workid": "00000000000000000000000000000000" +} From 0e85f29f10340734429674feb9800d6c7ef81f15 Mon Sep 17 00:00:00 2001 From: evan-forbes Date: Thu, 20 Aug 2026 17:43:30 -0500 Subject: [PATCH 05/22] fix(mining): harden optimistic block inventory --- crates/zakura-rpc/src/methods.rs | 7 ++- .../src/methods/types/get_block_template.rs | 30 +++++++++- .../methods/types/get_block_template/tests.rs | 15 ++++- .../src/methods/types/submit_block.rs | 55 ++++++++++++++++++- crates/zakurad/src/components/inbound.rs | 43 +++++++++------ .../zakurad/src/components/inbound/tests.rs | 32 ++++++++++- 6 files changed, 157 insertions(+), 25 deletions(-) diff --git a/crates/zakura-rpc/src/methods.rs b/crates/zakura-rpc/src/methods.rs index 781646f9fb..8a6b678b6c 100644 --- a/crates/zakura-rpc/src/methods.rs +++ b/crates/zakura-rpc/src/methods.rs @@ -1046,11 +1046,15 @@ where fn prepare_template_in_background(&self, template: &BlockTemplateResponse) { #[cfg(test)] { - let _ = template; + let _ = (template, self.gbt.try_acquire_template_preparation()); } #[cfg(not(test))] { + let Some(preparation_permit) = self.gbt.try_acquire_template_preparation() else { + metrics::counter!("mining.template_preparation.saturated").increment(1); + return; + }; let Ok(block) = proposal_block_from_template(template, None, &self.network) else { return; }; @@ -1061,6 +1065,7 @@ where let verifier = self.gbt.block_verifier_router(); tokio::spawn( async move { + let _preparation_permit = preparation_permit; if let Err(error) = verifier.oneshot(request).await { tracing::debug!(?error, "background mining candidate preparation failed"); } diff --git a/crates/zakura-rpc/src/methods/types/get_block_template.rs b/crates/zakura-rpc/src/methods/types/get_block_template.rs index db6e6fe366..3b90424f65 100644 --- a/crates/zakura-rpc/src/methods/types/get_block_template.rs +++ b/crates/zakura-rpc/src/methods/types/get_block_template.rs @@ -18,7 +18,7 @@ use derive_new::new; use jsonrpsee::core::RpcResult; use jsonrpsee_types::{ErrorCode, ErrorObject}; use rand::{rngs::OsRng, RngCore}; -use tokio::sync::mpsc; +use tokio::sync::{mpsc, OwnedSemaphorePermit, Semaphore}; use tower::{Service, ServiceExt}; use zcash_keys::address::Address; use zcash_protocol::memo::MemoBytes; @@ -65,6 +65,25 @@ pub use parameters::{ }; pub use proposal::{BlockProposalResponse, BlockTemplateTimeSource}; +const MAX_BACKGROUND_TEMPLATE_PREPARATIONS: usize = 1; + +#[derive(Clone, Debug)] +struct TemplatePreparationLimiter(Arc); + +impl Default for TemplatePreparationLimiter { + fn default() -> Self { + Self(Arc::new(Semaphore::new( + MAX_BACKGROUND_TEMPLATE_PREPARATIONS, + ))) + } +} + +impl TemplatePreparationLimiter { + fn try_acquire(&self) -> Option { + self.0.clone().try_acquire_owned().ok() + } +} + /// An alias to indicate that a usize value represents the depth of in-block dependencies of a /// transaction. /// @@ -577,6 +596,9 @@ where /// Whether state admission can trigger an early inventory. optimistic_block_inventory: bool, + + /// Limits detached template preparation work. + template_preparation_limiter: TemplatePreparationLimiter, } impl GetBlockTemplateHandler @@ -602,6 +624,7 @@ where .unwrap_or(SubmitBlockChannel::default().sender()), pending_blocks, optimistic_block_inventory, + template_preparation_limiter: TemplatePreparationLimiter::default(), } } @@ -635,6 +658,11 @@ where self.optimistic_block_inventory } + /// Reserves the background template preparation slot. + pub(crate) fn try_acquire_template_preparation(&self) -> Option { + self.template_preparation_limiter.try_acquire() + } + /// Randomizes the coinbase data, if miner parameters are set. pub fn randomize_coinbase_data(&mut self) { if let Some(miner_params) = &mut self.miner_params { diff --git a/crates/zakura-rpc/src/methods/types/get_block_template/tests.rs b/crates/zakura-rpc/src/methods/types/get_block_template/tests.rs index 307455e43f..3640eef12b 100644 --- a/crates/zakura-rpc/src/methods/types/get_block_template/tests.rs +++ b/crates/zakura-rpc/src/methods/types/get_block_template/tests.rs @@ -24,7 +24,20 @@ use zakura_chain::{ use crate::client::TransactionTemplate; use crate::config::mining::{default_miner_address, MinerAddressType}; -use super::MinerParams; +use super::{MinerParams, TemplatePreparationLimiter}; + +#[test] +fn template_preparation_is_single_flight() { + let limiter = TemplatePreparationLimiter::default(); + let permit = limiter + .try_acquire() + .expect("the first preparation reserves the slot"); + + assert!(limiter.try_acquire().is_none()); + + drop(permit); + assert!(limiter.try_acquire().is_some()); +} /// Tests transparent coinbase generation at every configured Sapling-and-later /// network upgrade activation. diff --git a/crates/zakura-rpc/src/methods/types/submit_block.rs b/crates/zakura-rpc/src/methods/types/submit_block.rs index 7ecee50621..672abe7a8d 100644 --- a/crates/zakura-rpc/src/methods/types/submit_block.rs +++ b/crates/zakura-rpc/src/methods/types/submit_block.rs @@ -6,7 +6,7 @@ use std::{ time::Duration, }; -use tokio::sync::{mpsc, oneshot, watch}; +use tokio::sync::{mpsc, oneshot, watch, OwnedSemaphorePermit, Semaphore}; use zakura_chain::block; @@ -39,6 +39,7 @@ pub struct SubmitBlockParameters { pub const PENDING_BLOCK_WAIT: Duration = Duration::from_secs(15); const MAX_PENDING_BLOCKS: usize = 16; +const MAX_PENDING_BLOCK_WAITS: usize = 32; /// A mined-block lifecycle event consumed by the block gossip task. #[derive(Debug)] @@ -86,9 +87,25 @@ struct PendingBlock { status: watch::Sender, } +/// Stores pending blocks and bounds peer waits. +#[derive(Debug)] +struct PendingBlockRegistryInner { + entries: Mutex>, + wait_permits: Arc, +} + /// Holds early-advertised block bodies until their contextual commits finish. -#[derive(Clone, Debug, Default)] -pub struct PendingBlockRegistry(Arc>>); +#[derive(Clone, Debug)] +pub struct PendingBlockRegistry(Arc); + +impl Default for PendingBlockRegistry { + fn default() -> Self { + Self(Arc::new(PendingBlockRegistryInner { + entries: Mutex::new(HashMap::new()), + wait_permits: Arc::new(Semaphore::new(MAX_PENDING_BLOCK_WAITS)), + })) + } +} impl PendingBlockRegistry { /// Inserts a block before its early inventory is sent. @@ -98,6 +115,7 @@ impl PendingBlockRegistry { let hash = block.hash(); let mut entries = self .0 + .entries .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); if entries.contains_key(&hash) { @@ -117,6 +135,7 @@ impl PendingBlockRegistry { pub fn resolve(&self, hash: block::Hash, result: Result, ()>) { let entry = self .0 + .entries .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .remove(&hash); @@ -137,14 +156,26 @@ impl PendingBlockRegistry { ) -> impl std::future::Future>> + Send + 'static { let status = self .0 + .entries .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .get(&hash) .map(|entry| entry.status.subscribe()); + let wait_permit = status.as_ref().and_then(|_| { + self.0 + .wait_permits + .clone() + .try_acquire_owned() + .map_err(|_| { + metrics::counter!("mining.pending_peer_wait.saturated").increment(1); + }) + .ok() + }); let deadline = tokio::time::Instant::now() + PENDING_BLOCK_WAIT; async move { let mut status = status?; + let _wait_permit: OwnedSemaphorePermit = wait_permit?; let start = std::time::Instant::now(); let result = tokio::time::timeout_at(deadline, async { loop { @@ -308,6 +339,24 @@ mod tests { assert_eq!(wait.await.expect("wait task succeeds"), None); } + #[tokio::test] + async fn pending_block_waits_are_bounded() { + let registry = PendingBlockRegistry::default(); + let block = test_block(); + let hash = block.hash(); + assert!(registry.insert(block.clone())); + + let waits: Vec<_> = (0..MAX_PENDING_BLOCK_WAITS) + .map(|_| registry.wait(hash)) + .collect(); + assert_eq!(registry.wait(hash).await, None); + + drop(waits); + let wait = registry.wait(hash); + registry.resolve(hash, Ok(block.clone())); + assert_eq!(wait.await, Some(block)); + } + #[test] fn pending_registry_is_bounded() { let registry = PendingBlockRegistry::default(); diff --git a/crates/zakurad/src/components/inbound.rs b/crates/zakurad/src/components/inbound.rs index dbcdbd58a2..709ef841f9 100644 --- a/crates/zakurad/src/components/inbound.rs +++ b/crates/zakurad/src/components/inbound.rs @@ -197,6 +197,28 @@ async fn retained_block_height(mut state: State, hash: block::Hash) -> Option Result>, zn::BoxError> { + // Subscribe before the state lookup. A commit can remove the registry entry while state + // answers this request. + let pending_wait = pending_blocks.wait(hash); + let response = state + .ready() + .await? + .call(zs::Request::AnyChainBlock(hash.into())) + .await?; + + match response { + zs::Response::Block(Some(block)) => Ok(Some(block)), + zs::Response::Block(None) => Ok(pending_wait.await), + _ => unreachable!("wrong response from state"), + } +} + fn mempool_queue_source(source: zn::PeerSource) -> mempool::QueueSource { match source { zn::PeerSource::LegacySocket(addr) => mempool::QueueSource::LegacySocket(*addr), @@ -623,24 +645,9 @@ impl Service for Inbound { let state = state.clone(); let pending_blocks = pending_blocks.clone(); lookups.push(async move { - // Subscribe before the state lookup. A commit can complete and - // remove the registry entry while state answers this request. - let pending_wait = pending_blocks.wait(hash); - let response = state - .clone() - .ready() - .await? - .call(zs::Request::Block(hash.into())) - .await?; - match response { - zs::Response::Block(Some(block)) => { - Ok::<_, zn::BoxError>((index, hash, Some(block))) - } - zs::Response::Block(None) => { - Ok((index, hash, pending_wait.await)) - } - _ => unreachable!("wrong response from state"), - } + let block = + block_by_hash_or_pending(state, pending_blocks, hash).await?; + Ok::<_, zn::BoxError>((index, hash, block)) }); } diff --git a/crates/zakurad/src/components/inbound/tests.rs b/crates/zakurad/src/components/inbound/tests.rs index e89a9f1519..d1610425cb 100644 --- a/crates/zakurad/src/components/inbound/tests.rs +++ b/crates/zakurad/src/components/inbound/tests.rs @@ -6,10 +6,40 @@ use std::{ }; use super::{ - block_misbehavior, canonical_ip, PrunedBlockNotFoundLogger, + block_by_hash_or_pending, block_misbehavior, canonical_ip, PrunedBlockNotFoundLogger, ZCASHD_COMPAT_PRUNED_BLOCK_LOG_INTERVAL, }; +#[tokio::test] +async fn peer_block_lookup_queries_all_active_chains() { + use std::sync::Arc; + + use tower::{buffer::Buffer, util::BoxService}; + use zakura_chain::{block::Block, serialization::ZcashDeserializeInto}; + use zakura_rpc::PendingBlockRegistry; + + let block: Arc = zakura_test::vectors::BLOCK_MAINNET_GENESIS_BYTES + .zcash_deserialize_into() + .expect("the genesis block is valid"); + let hash = block.hash(); + let expected_block = block.clone(); + let state = tower::service_fn(move |request| { + let expected_block = expected_block.clone(); + async move { + assert_eq!(request, zakura_state::Request::AnyChainBlock(hash.into())); + Ok::<_, zakura_state::BoxError>(zakura_state::Response::Block(Some(expected_block))) + } + }); + let state = Buffer::new(BoxService::new(state), 1); + + assert_eq!( + block_by_hash_or_pending(state, PendingBlockRegistry::default(), hash) + .await + .expect("the state lookup succeeds"), + Some(block), + ); +} + mod fake_peer_set; mod real_peer_set; From 863e443385d2fea954337d42519f2bba5bc5806d Mon Sep 17 00:00:00 2001 From: evan-forbes Date: Fri, 21 Aug 2026 21:05:35 -0500 Subject: [PATCH 06/22] chore(rpc): regenerate OpenRPC artifact Main added the checked-in `rpc_openrpc.rs` artifact and its staleness check in #764. This branch edits the `getblocktemplate` and `submitblock` doc comments that the artifact derives from, so `cargo xtask check-rpc-artifacts` fails after rebasing onto main. Regenerate the artifact so it matches the doc comments. --- crates/zakura-rpc/src/methods/rpc_openrpc.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/zakura-rpc/src/methods/rpc_openrpc.rs b/crates/zakura-rpc/src/methods/rpc_openrpc.rs index fa9be89434..cd28c9eb8d 100644 --- a/crates/zakura-rpc/src/methods/rpc_openrpc.rs +++ b/crates/zakura-rpc/src/methods/rpc_openrpc.rs @@ -153,7 +153,7 @@ pub static METHODS: ::phf::Map<&str, openrpsee::openrpc::RpcMethod> = ::phf::phf deprecated: false, }, "getblocktemplate" => openrpsee::openrpc::RpcMethod { - description: "Returns a block template for mining new Zcash blocks.\n\n# Parameters\n\n- `jsonrequestobject`: (string, optional) A JSON object containing arguments.\n\nzcashd reference: [`getblocktemplate`](https://zcash-rpc.github.io/getblocktemplate.html)\nmethod: post\ntags: mining\n\n# Notes\n\nArguments to this RPC are currently ignored.\nLong polling, block proposals, server lists, and work IDs are not supported.\n\nMiners can make arbitrary changes to blocks, as long as:\n- the data sent to `submitblock` is a valid Zcash block, and\n- the parent block is a valid block that Zebra already has, or will receive soon.\n\nZebra verifies blocks in parallel, and keeps recent chains in parallel,\nso moving between chains and forking chains is very cheap.\n", + description: "Returns a block template for mining new Zcash blocks.\n\n# Parameters\n\n- `jsonrequestobject`: (string, optional) A JSON object containing arguments.\n\nzcashd reference: [`getblocktemplate`](https://zcash-rpc.github.io/getblocktemplate.html)\nmethod: post\ntags: mining\n\n# Notes\n\nServer lists are not supported. Long polling, block proposals, and work IDs are supported.\n\nMiners can make arbitrary changes to blocks, as long as:\n- the data sent to `submitblock` is a valid Zcash block, and\n- the parent block is a valid block that Zebra already has, or will receive soon.\n\nZebra verifies blocks in parallel, and keeps recent chains in parallel,\nso moving between chains and forking chains is very cheap.\n", params: |_g| vec![ _g.param::("parameters", crate::methods::PARAM_PARAMETERS_DESC, false), ], @@ -161,7 +161,7 @@ pub static METHODS: ::phf::Map<&str, openrpsee::openrpc::RpcMethod> = ::phf::phf deprecated: false, }, "submitblock" => openrpsee::openrpc::RpcMethod { - description: "Submits block to the node to be validated and committed.\nReturns the [`SubmitBlockResponse`] for the operation, as a JSON string.\n\nzcashd reference: [`submitblock`](https://zcash.github.io/rpc/submitblock.html)\nmethod: post\ntags: mining\n\n# Parameters\n\n- `hexdata`: (string, required)\n- `jsonparametersobject`: (string, optional) - currently ignored\n\n# Notes\n\n - `jsonparametersobject` holds a single field, workid, that must be included in submissions if provided by the server.\n", + description: "Submits block to the node to be validated and committed.\nReturns the [`SubmitBlockResponse`] for the operation, as a JSON string.\n\nzcashd reference: [`submitblock`](https://zcash.github.io/rpc/submitblock.html)\nmethod: post\ntags: mining\n\n# Parameters\n\n- `hexdata`: (string, required)\n- `jsonparametersobject`: (string, optional)\n\n# Notes\n\n - `jsonparametersobject` holds a single field, workid, that must be included in submissions if provided by the server.\n", params: |_g| vec![ _g.param::("hex_data", crate::methods::PARAM_HEX_DATA_DESC, true), _g.param::("_parameters", crate::methods::PARAM__PARAMETERS_DESC, false), From 7dc22127fc91510b87a6fdacd4e444f9c75f4302 Mon Sep 17 00:00:00 2001 From: evan-forbes Date: Sun, 23 Aug 2026 13:33:42 -0500 Subject: [PATCH 07/22] fix(mining): require expected work before optimistic relay --- crates/zakura-consensus/src/block.rs | 48 +++- crates/zakura-rpc/src/config/mining.rs | 3 +- crates/zakura-rpc/src/methods.rs | 6 +- crates/zakura-state/src/lib.rs | 9 +- crates/zakura-state/src/request.rs | 51 ++++ crates/zakura-state/src/response.rs | 20 ++ crates/zakura-state/src/service.rs | 84 ++++++- crates/zakura-state/src/service/check.rs | 32 ++- crates/zakura-state/src/service/tests.rs | 292 ++++++++++++++++++++++- docs/changelog/unreleased/748.md | 3 +- 10 files changed, 525 insertions(+), 23 deletions(-) diff --git a/crates/zakura-consensus/src/block.rs b/crates/zakura-consensus/src/block.rs index 9667c6972d..e5a829b0be 100644 --- a/crates/zakura-consensus/src/block.rs +++ b/crates/zakura-consensus/src/block.rs @@ -329,12 +329,19 @@ where prepared_block.block = block; prepared_block.hash = hash; prepared_block.height = height; - return commit_prepared_block( - state_service, - prepared_block, - request.admission(), - ) - .await; + let admission = request.admission(); + if let Some(admission) = &admission { + if check_prepared_mined_relay_eligibility( + &mut state_service, + (&prepared_block).into(), + ) + .await? + == zs::PreparedMinedRelayEligibility::Authorized + { + admission.authorize_optimistic_relay(); + } + } + return commit_prepared_block(state_service, prepared_block, admission).await; } metrics::histogram!("mining.solved_header_check.duration_seconds") .record(solved_header_start.elapsed().as_secs_f64()); @@ -520,6 +527,35 @@ where } } +async fn check_prepared_mined_relay_eligibility( + state_service: &mut S, + block: zs::BlockCommitmentData, +) -> Result +where + S: Service + Send + Clone + 'static, + S::Future: Send + 'static, +{ + let hash = block.block.hash(); + let preflight_start = std::time::Instant::now(); + let response = async { + state_service + .ready() + .await + .map_err(|source| VerifyBlockError::StateService { source, hash })? + .call(zs::Request::CheckPreparedMinedRelayEligibility(block)) + .await + .map_err(|source| map_commit_error(source, hash)) + } + .await; + metrics::histogram!("mining.prepared_relay_preflight.duration_seconds") + .record(preflight_start.elapsed().as_secs_f64()); + + match response? { + zs::Response::PreparedMinedRelayEligibility(eligibility) => Ok(eligibility), + _ => unreachable!("wrong response for prepared mined-block relay eligibility"), + } +} + async fn commit_prepared_block( mut state_service: S, prepared_block: zs::SemanticallyVerifiedBlock, diff --git a/crates/zakura-rpc/src/config/mining.rs b/crates/zakura-rpc/src/config/mining.rs index 97a82ab0b3..c4742c1342 100644 --- a/crates/zakura-rpc/src/config/mining.rs +++ b/crates/zakura-rpc/src/config/mining.rs @@ -64,7 +64,8 @@ pub struct Config { #[serde(default)] pub internal_miner: bool, - /// Advertise mined block hashes after state admission and before contextual commit completes. + /// Advertise prepared mined block hashes after expected-work validation and state admission, + /// but before contextual commit completes. pub optimistic_block_inventory: bool, } diff --git a/crates/zakura-rpc/src/methods.rs b/crates/zakura-rpc/src/methods.rs index 8a6b678b6c..cb6abe3610 100644 --- a/crates/zakura-rpc/src/methods.rs +++ b/crates/zakura-rpc/src/methods.rs @@ -2833,7 +2833,11 @@ where admitted = admission.wait() => { metrics::histogram!("mining.state_admission.duration_seconds") .record(admission_start.elapsed().as_secs_f64()); - if admitted && optimistic_block_inventory && pending_blocks.insert(block.clone()) { + if admitted + && admission.optimistic_relay_authorized() + && optimistic_block_inventory + && pending_blocks.insert(block.clone()) + { let (advertised, receiver) = tokio::sync::oneshot::channel(); let event = MinedBlockEvent::Early { hash: block_hash, diff --git a/crates/zakura-state/src/lib.rs b/crates/zakura-state/src/lib.rs index 27fe2018bb..e40440f533 100644 --- a/crates/zakura-state/src/lib.rs +++ b/crates/zakura-state/src/lib.rs @@ -56,9 +56,10 @@ pub use error::{ }; pub use header_chain::*; pub use request::{ - BlockAdmission, CheckpointVerifiedBlock, CommitSemanticallyVerifiedBlockRequest, HashOrHeight, - HeaderChainBodyEvidenceAuthority, MappedRequest, PreparedHeaderChainBodyEvidence, - PreparedHeaderChainInsert, ReadRequest, Request, SemanticallyVerifiedBlock, + BlockAdmission, BlockCommitmentData, CheckpointVerifiedBlock, + CommitSemanticallyVerifiedBlockRequest, HashOrHeight, HeaderChainBodyEvidenceAuthority, + MappedRequest, PreparedHeaderChainBodyEvidence, PreparedHeaderChainInsert, ReadRequest, + Request, SemanticallyVerifiedBlock, }; #[cfg(feature = "indexer")] @@ -66,7 +67,7 @@ pub use request::Spend; pub use response::{ AnyTx, BlockSyncBodyMetadata, GetBlockTemplateChainInfo, KnownBlock, MinedTx, - NonFinalizedBlocksListener, ReadResponse, Response, + NonFinalizedBlocksListener, PreparedMinedRelayEligibility, ReadResponse, Response, }; #[cfg(any(test, feature = "header-fuzz"))] pub use service::finalized_state::{replay_recovery_rows_bytes, RecoveryRowsReplaySummary}; diff --git a/crates/zakura-state/src/request.rs b/crates/zakura-state/src/request.rs index 21e72ce4c6..548cdcdbe7 100644 --- a/crates/zakura-state/src/request.rs +++ b/crates/zakura-state/src/request.rs @@ -48,6 +48,7 @@ pub struct BlockAdmission(Arc); #[derive(Debug)] struct BlockAdmissionInner { state: AtomicU8, + optimistic_relay_authorized: AtomicBool, changed: Notify, } @@ -60,10 +61,24 @@ impl BlockAdmission { pub fn pending() -> Self { Self(Arc::new(BlockAdmissionInner { state: AtomicU8::new(Self::PENDING), + optimistic_relay_authorized: AtomicBool::new(false), changed: Notify::new(), })) } + /// Authorizes optimistic relay if state later admits the prepared mined block. + #[doc(hidden)] + pub fn authorize_optimistic_relay(&self) { + self.0 + .optimistic_relay_authorized + .store(true, Ordering::Release); + } + + /// Returns true when consensus authorized optimistic relay for this admission. + pub fn optimistic_relay_authorized(&self) -> bool { + self.0.optimistic_relay_authorized.load(Ordering::Acquire) + } + /// Marks the block as admitted to the active non-finalized write queue. pub(crate) fn admit(&self) { if self @@ -366,6 +381,24 @@ pub struct SemanticallyVerifiedBlock { pub auth_data_root: Option, } +/// Data required to check a prepared mined block before optimistic relay. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BlockCommitmentData { + /// The block whose header commits to the prepared body and parent history. + pub block: Arc, + /// The precomputed authorizing-data commitment root, when available. + pub auth_data_root: Option, +} + +impl From<&SemanticallyVerifiedBlock> for BlockCommitmentData { + fn from(block: &SemanticallyVerifiedBlock) -> Self { + Self { + block: block.block.clone(), + auth_data_root: block.auth_data_root, + } + } +} + /// A block ready to be committed directly to the finalized state with /// a small number of checks if compared with a `ContextuallyVerifiedBlock`. /// @@ -803,6 +836,9 @@ mod tests { #[tokio::test] async fn block_admission_keeps_its_first_terminal_state() { let rejected = BlockAdmission::pending(); + assert!(!rejected.optimistic_relay_authorized()); + rejected.authorize_optimistic_relay(); + assert!(rejected.optimistic_relay_authorized()); rejected.reject(); rejected.admit(); assert!(!rejected.wait().await); @@ -1411,6 +1447,9 @@ pub enum Request { /// Returns [`Response::ValidBestChainTipNullifiersAndAnchors`] CheckBestChainTipNullifiersAndAnchors(UnminedTx), + /// Checks the expected work, body commitment, parent history, and selected tip. + CheckPreparedMinedRelayEligibility(BlockCommitmentData), + /// Calculates the median-time-past for the *next* block on the best chain. /// /// Returns [`Response::BestChainNextMedianTimePast`] when successful. @@ -1491,6 +1530,9 @@ impl Request { Request::CheckBestChainTipNullifiersAndAnchors(_) => { "best_chain_tip_nullifiers_anchors" } + Request::CheckPreparedMinedRelayEligibility(_) => { + "check_prepared_mined_relay_eligibility" + } Request::BestChainNextMedianTimePast => "best_chain_next_median_time_past", Request::BestChainBlockHash(_) => "best_chain_block_hash", Request::KnownBlock(_) => "known_block", @@ -1930,6 +1972,9 @@ pub enum ReadRequest { /// Returns [`ReadResponse::ValidBestChainTipNullifiersAndAnchors`]. CheckBestChainTipNullifiersAndAnchors(UnminedTx), + /// Checks the expected work, body commitment, parent history, and selected tip. + CheckPreparedMinedRelayEligibility(BlockCommitmentData), + /// Calculates the median-time-past for the *next* block on the best chain. /// /// Returns [`ReadResponse::BestChainNextMedianTimePast`] when successful. @@ -2040,6 +2085,9 @@ impl ReadRequest { ReadRequest::CheckBestChainTipNullifiersAndAnchors(_) => { "best_chain_tip_nullifiers_anchors" } + ReadRequest::CheckPreparedMinedRelayEligibility(_) => { + "check_prepared_mined_relay_eligibility" + } ReadRequest::BestChainNextMedianTimePast => "best_chain_next_median_time_past", ReadRequest::BestChainBlockHash(_) => "best_chain_block_hash", #[cfg(feature = "indexer")] @@ -2098,6 +2146,9 @@ impl TryFrom for ReadRequest { Request::CheckBestChainTipNullifiersAndAnchors(tx) => { Ok(ReadRequest::CheckBestChainTipNullifiersAndAnchors(tx)) } + Request::CheckPreparedMinedRelayEligibility(block) => { + Ok(ReadRequest::CheckPreparedMinedRelayEligibility(block)) + } Request::ApplyHeaderChainInsert { .. } | Request::RecordHeaderChainBodyUnavailable { .. } diff --git a/crates/zakura-state/src/response.rs b/crates/zakura-state/src/response.rs index da08e764bd..345d0c1e59 100644 --- a/crates/zakura-state/src/response.rs +++ b/crates/zakura-state/src/response.rs @@ -34,6 +34,17 @@ use crate::{ #[cfg(test)] mod tests; +/// State's decision for a prepared mined block's optimistic relay. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PreparedMinedRelayEligibility { + /// The block proves expected work and extends the selected tip. + Authorized, + /// The block proves expected work but does not extend the selected tip. + CommitFirst, + /// State cannot prove expected work from the available context. + Unavailable, +} + #[derive(Clone, Debug, PartialEq, Eq)] /// A response to a [`StateService`](crate::service::StateService) [`Request`]. pub enum Response { @@ -113,6 +124,9 @@ pub enum Response { /// Does not check transparent UTXO inputs ValidBestChainTipNullifiersAndAnchors, + /// Response to [`Request::CheckPreparedMinedRelayEligibility`]. + PreparedMinedRelayEligibility(PreparedMinedRelayEligibility), + /// Response to [`Request::BestChainNextMedianTimePast`]. /// Contains the median-time-past for the *next* block on the best chain. BestChainNextMedianTimePast(DateTime32), @@ -579,6 +593,9 @@ pub enum ReadResponse { /// Does not check transparent UTXO inputs ValidBestChainTipNullifiersAndAnchors, + /// Response to [`ReadRequest::CheckPreparedMinedRelayEligibility`]. + PreparedMinedRelayEligibility(PreparedMinedRelayEligibility), + /// Response to [`ReadRequest::BestChainNextMedianTimePast`]. /// Contains the median-time-past for the *next* block on the best chain. BestChainNextMedianTimePast(DateTime32), @@ -691,6 +708,9 @@ impl TryFrom for Response { ReadResponse::BlockHeaders(headers) => Ok(Response::BlockHeaders(headers)), ReadResponse::ValidBestChainTipNullifiersAndAnchors => Ok(Response::ValidBestChainTipNullifiersAndAnchors), + ReadResponse::PreparedMinedRelayEligibility(eligibility) => { + Ok(Response::PreparedMinedRelayEligibility(eligibility)) + } ReadResponse::UsageInfo(_) | ReadResponse::PruningInfo { .. } diff --git a/crates/zakura-state/src/service.rs b/crates/zakura-state/src/service.rs index 46c2e32faa..8d854d2ab2 100644 --- a/crates/zakura-state/src/service.rs +++ b/crates/zakura-state/src/service.rs @@ -63,9 +63,10 @@ use crate::{ read::find, watch_receiver::WatchReceiver, }, - BlockAdmission, BoxError, CheckpointVerifiedBlock, CommitSemanticallyVerifiedError, Config, - HashOrHeight, HistoricalTreeUnavailable, KnownBlock, ReadRequest, ReadResponse, Request, - Response, SemanticallyVerifiedBlock, StateInitError, + BlockAdmission, BlockCommitmentData, BoxError, CheckpointVerifiedBlock, + CommitSemanticallyVerifiedError, Config, HashOrHeight, HistoricalTreeUnavailable, KnownBlock, + PreparedMinedRelayEligibility, ReadRequest, ReadResponse, Request, Response, + SemanticallyVerifiedBlock, StateInitError, }; pub mod block_iter; @@ -1797,6 +1798,7 @@ impl Service for StateService { | Request::FindBlockHashes { .. } | Request::FindBlockHeaders { .. } | Request::CheckBestChainTipNullifiersAndAnchors(_) + | Request::CheckPreparedMinedRelayEligibility(_) | Request::CheckBlockProposalValidity(_) => { // Redirect the request to the concurrent ReadStateService let read_service = self.read_service.clone(); @@ -3009,6 +3011,18 @@ impl Service for ReadStateService { Ok(ReadResponse::ValidBestChainTipNullifiersAndAnchors) } + ReadRequest::CheckPreparedMinedRelayEligibility(commitment) => { + let latest_non_finalized_state = state.latest_non_finalized_state(); + let eligibility = check_prepared_mined_relay_eligibility_for_state( + &state.network, + &latest_non_finalized_state, + &state.db, + commitment, + )?; + + Ok(ReadResponse::PreparedMinedRelayEligibility(eligibility)) + } + // Used by the get_block and get_block_hash RPCs. ReadRequest::BestChainBlockHash(height) => Ok(ReadResponse::BlockHash( read::hash_by_height(state.latest_best_chain(), &state.db, height), @@ -3147,6 +3161,70 @@ impl Service for ReadStateService { } } +fn check_prepared_mined_relay_eligibility_for_state( + network: &Network, + non_finalized_state: &NonFinalizedState, + db: &ZakuraDb, + commitment: BlockCommitmentData, +) -> Result { + let parent_hash = commitment.block.header.previous_block_hash; + let parent_chain = + non_finalized_state.find_chain(|chain| chain.contains_block_hash(parent_hash)); + let history_tree = read::tree::history_tree(parent_chain, db, parent_hash.into()); + let history_tree = match history_tree { + Some(history_tree) => history_tree, + None if matches!( + commitment.block.commitment(network)?, + block::Commitment::PreSaplingReserved(_) + | block::Commitment::FinalSaplingRoot(_) + | block::Commitment::ChainHistoryActivationReserved + ) => + { + Arc::new(zakura_chain::history_tree::HistoryTree::default()) + } + None => return Ok(PreparedMinedRelayEligibility::Unavailable), + }; + check::block_commitment_is_valid_for_chain_history( + commitment.block.clone(), + network, + &history_tree, + commitment.auth_data_root, + )?; + + let relevant_chain: Vec<_> = + any_ancestor_blocks(non_finalized_state, db, parent_hash).collect(); + if relevant_chain.is_empty() { + return Ok(PreparedMinedRelayEligibility::Unavailable); + } + let candidate_height = commitment + .block + .coinbase_height() + .ok_or(crate::ValidateContextError::NotReadyToBeCommitted)?; + let finalized_tip_height = db.finalized_tip_height().or_else(|| { + relevant_chain + .last() + .and_then(|block| block.coinbase_height()) + }); + check::block_is_valid_for_recent_chain_data( + &commitment.block, + candidate_height, + network, + finalized_tip_height, + relevant_chain, + )?; + + if network.disable_pow() { + return Ok(PreparedMinedRelayEligibility::Unavailable); + } + let extends_selected_tip = read::best_tip(non_finalized_state, db) + .is_some_and(|(_, selected_tip_hash)| selected_tip_hash == parent_hash); + if extends_selected_tip { + Ok(PreparedMinedRelayEligibility::Authorized) + } else { + Ok(PreparedMinedRelayEligibility::CommitFirst) + } +} + /// Initialize a state service from the provided [`Config`]. /// Returns a boxed state service, a read-only state service, /// and receivers for state chain tip updates. diff --git a/crates/zakura-state/src/service/check.rs b/crates/zakura-state/src/service/check.rs index d11c826b5f..c075ac438b 100644 --- a/crates/zakura-state/src/service/check.rs +++ b/crates/zakura-state/src/service/check.rs @@ -57,6 +57,28 @@ pub(crate) fn block_is_valid_for_recent_chain( finalized_tip_height: Option, relevant_chain: C, ) -> Result<(), ValidateContextError> +where + C: IntoIterator, + C::Item: Borrow, + C::IntoIter: ExactSizeIterator, +{ + block_is_valid_for_recent_chain_data( + &semantically_verified.block, + semantically_verified.height, + network, + finalized_tip_height, + relevant_chain, + ) +} + +/// Checks the recent-chain rules required for prepared mined-block relay. +pub(crate) fn block_is_valid_for_recent_chain_data( + candidate_block: &Block, + candidate_height: block::Height, + network: &Network, + finalized_tip_height: Option, + relevant_chain: C, +) -> Result<(), ValidateContextError> where C: IntoIterator, C::Item: Borrow, @@ -64,7 +86,7 @@ where { let finalized_tip_height = finalized_tip_height .expect("finalized state must contain at least one block to do contextual validation"); - check::block_is_not_orphaned(finalized_tip_height, semantically_verified.height)?; + check::block_is_not_orphaned(finalized_tip_height, candidate_height)?; let relevant_chain: Vec<_> = relevant_chain .into_iter() @@ -73,7 +95,7 @@ where let Some(parent_block) = relevant_chain.first() else { warn!( - ?semantically_verified, + ?candidate_height, ?finalized_tip_height, "state must contain parent block to do contextual validation" ); @@ -85,7 +107,7 @@ where let parent_height = parent_block .coinbase_height() .expect("valid blocks have a coinbase height"); - check::height_one_more_than_parent_height(parent_height, semantically_verified.height)?; + check::height_one_more_than_parent_height(parent_height, candidate_height)?; // skip this check during tests if we don't have enough blocks in the chain // process_queued also checks the chain length, so we can skip this assertion during testing @@ -126,10 +148,10 @@ where ) }); let difficulty_adjustment = - AdjustedDifficulty::new_from_block(&semantically_verified.block, network, relevant_data) + AdjustedDifficulty::new_from_block(candidate_block, network, relevant_data) .map_err(|_| ValidateContextError::NotReadyToBeCommitted)?; check::difficulty_threshold_and_time_are_valid( - semantically_verified.block.header.difficulty_threshold, + candidate_block.header.difficulty_threshold, difficulty_adjustment, )?; diff --git a/crates/zakura-state/src/service/tests.rs b/crates/zakura-state/src/service/tests.rs index 02f571d749..796668c4d7 100644 --- a/crates/zakura-state/src/service/tests.rs +++ b/crates/zakura-state/src/service/tests.rs @@ -32,12 +32,300 @@ use crate::{ }, tests::setup::{partial_nu5_chain_strategy, transaction_v4_from_coinbase}, BoxError, CheckpointVerifiedBlock, Config, HistoricalTreeUnavailable, PruningConfig, Request, - Response, SemanticallyVerifiedBlock, StateInitError, StorageMode, CHAIN_TIP_UPDATE_WAIT_LIMIT, - MAX_HISTORICAL_TREE_REPLAY_BLOCKS, + Response, SemanticallyVerifiedBlock, StateInitError, StorageMode, ValidateContextError, + CHAIN_TIP_UPDATE_WAIT_LIMIT, MAX_HISTORICAL_TREE_REPLAY_BLOCKS, }; const LAST_BLOCK_HEIGHT: u32 = 10; +fn prepared_relay_test_state() -> ( + Network, + super::finalized_state::FinalizedState, + super::non_finalized_state::NonFinalizedState, + Arc, +) { + use crate::tests::FakeChainHelper; + + let network = Network::Mainnet; + let heartwood_height = NetworkUpgrade::Heartwood + .activation_height(&network) + .expect("Heartwood activates") + .0; + let root = Arc::new( + network.block_map()[&(heartwood_height - 1)] + .zcash_deserialize_into::() + .expect("pre-Heartwood test block is valid"), + ); + let finalized = super::finalized_state::FinalizedState::new(&Config::ephemeral(), &network) + .expect("ephemeral finalized state opens"); + let mut non_finalized = super::non_finalized_state::NonFinalizedState::new(&network); + non_finalized + .commit_new_chain(root.clone().prepare(), &finalized) + .expect("root commits"); + let activation = root.make_fake_child().set_block_commitment([0; 32]); + non_finalized + .commit_block(activation.clone().prepare(), &finalized) + .expect("Heartwood activation commits"); + let sibling_commitment: [u8; 32] = non_finalized + .best_chain() + .expect("activation chain exists") + .history_block_commitment_tree() + .hash() + .expect("activation creates a history root") + .into(); + let best = activation + .make_fake_child() + .set_block_commitment(sibling_commitment) + .set_work(100); + let side = activation + .make_fake_child() + .set_block_commitment(sibling_commitment) + .set_work(50); + non_finalized + .commit_block(best.prepare(), &finalized) + .expect("best child commits"); + non_finalized + .commit_block(side.clone().prepare(), &finalized) + .expect("side child commits"); + + (network, finalized, non_finalized, side) +} + +fn prepared_relay_difficulty_context() -> ( + Network, + super::finalized_state::FinalizedState, + super::non_finalized_state::NonFinalizedState, + Arc, +) { + use crate::tests::FakeChainHelper; + use zakura_header_chain::POW_ADJUSTMENT_BLOCK_SPAN; + + let network = Network::Mainnet; + let heartwood_height = NetworkUpgrade::Heartwood + .activation_height(&network) + .expect("Heartwood activates") + .0; + let root = Arc::new( + network.block_map()[&(heartwood_height - 1)] + .zcash_deserialize_into::() + .expect("pre-Heartwood test block is valid"), + ); + let finalized = super::finalized_state::FinalizedState::new(&Config::ephemeral(), &network) + .expect("ephemeral finalized state opens"); + let mut non_finalized = super::non_finalized_state::NonFinalizedState::new(&network); + non_finalized + .commit_new_chain(root.clone().prepare(), &finalized) + .expect("root commits"); + let mut tip = root; + for context_index in 0..POW_ADJUSTMENT_BLOCK_SPAN { + let commitment = if context_index == 0 { + [0; 32] + } else { + non_finalized + .best_chain() + .expect("the context chain exists") + .history_block_commitment_tree() + .hash() + .expect("the context chain has a history root") + .into() + }; + let mut child = tip.make_fake_child().set_block_commitment(commitment); + let child_height = child.coinbase_height().expect("the child has a height"); + Arc::make_mut(&mut Arc::make_mut(&mut child).header).time = + tip.header.time + NetworkUpgrade::target_spacing_for_height(&network, child_height); + non_finalized + .commit_block(child.clone().prepare(), &finalized) + .expect("difficulty context block commits"); + tip = child; + } + + (network, finalized, non_finalized, tip) +} + +#[test] +fn prepared_relay_preflight_authorizes_a_selected_tip_child() { + use crate::tests::FakeChainHelper; + + let _init_guard = zakura_test::init(); + let (network, finalized, non_finalized, _) = prepared_relay_test_state(); + let best = non_finalized + .best_tip_block() + .expect("the test state has a best tip") + .block + .clone(); + let commitment: [u8; 32] = non_finalized + .best_chain() + .expect("the best chain exists") + .history_block_commitment_tree() + .hash() + .expect("the best chain has a history root") + .into(); + let child = best.make_fake_child().set_block_commitment(commitment); + + let eligibility = super::check_prepared_mined_relay_eligibility_for_state( + &network, + &non_finalized, + &finalized.db, + crate::BlockCommitmentData { + block: child, + auth_data_root: None, + }, + ) + .expect("the selected tip child passes the relay preflight"); + + assert_eq!( + eligibility, + crate::PreparedMinedRelayEligibility::Authorized + ); +} + +#[test] +fn prepared_relay_preflight_uses_commit_first_for_a_side_chain() { + use crate::tests::FakeChainHelper; + + let _init_guard = zakura_test::init(); + let (network, finalized, non_finalized, side) = prepared_relay_test_state(); + let parent_hash = side.hash(); + let parent_chain = non_finalized + .find_chain(|chain| chain.contains_block_hash(parent_hash)) + .expect("side parent chain exists"); + let history_tree = + super::read::tree::history_tree(Some(parent_chain), &finalized.db, parent_hash.into()) + .expect("side parent has a history tree"); + let commitment: [u8; 32] = history_tree + .hash() + .expect("the side chain has a history root") + .into(); + let child = side.make_fake_child().set_block_commitment(commitment); + + let eligibility = super::check_prepared_mined_relay_eligibility_for_state( + &network, + &non_finalized, + &finalized.db, + crate::BlockCommitmentData { + block: child, + auth_data_root: None, + }, + ) + .expect("the side-chain child proves its expected work"); + + assert_eq!( + eligibility, + crate::PreparedMinedRelayEligibility::CommitFirst + ); +} + +#[test] +fn prepared_relay_preflight_rejects_an_easier_claimed_target() { + use crate::tests::FakeChainHelper; + + let _init_guard = zakura_test::init(); + let (network, finalized, non_finalized, tip) = prepared_relay_difficulty_context(); + let commitment: [u8; 32] = non_finalized + .best_chain() + .expect("the best chain exists") + .history_block_commitment_tree() + .hash() + .expect("the best chain has a history root") + .into(); + let mut child = tip + .make_fake_child() + .set_block_commitment(commitment) + .set_work(1); + let child_height = child.coinbase_height().expect("the child has a height"); + Arc::make_mut(&mut Arc::make_mut(&mut child).header).time = + tip.header.time + NetworkUpgrade::target_spacing_for_height(&network, child_height); + + let error = super::check_prepared_mined_relay_eligibility_for_state( + &network, + &non_finalized, + &finalized.db, + crate::BlockCommitmentData { + block: child, + auth_data_root: None, + }, + ) + .expect_err("an easier claimed target fails the relay preflight"); + + assert!(matches!( + error.downcast_ref::(), + Some(ValidateContextError::InvalidDifficultyThreshold { .. }) + )); +} + +#[test] +fn prepared_relay_preflight_rejects_time_at_or_below_median() { + use crate::tests::FakeChainHelper; + use zakura_header_chain::{AdjustedDifficulty, POW_ADJUSTMENT_BLOCK_SPAN}; + + let _init_guard = zakura_test::init(); + let (network, finalized, non_finalized, tip) = prepared_relay_difficulty_context(); + let commitment: [u8; 32] = non_finalized + .best_chain() + .expect("the best chain exists") + .history_block_commitment_tree() + .hash() + .expect("the best chain has a history root") + .into(); + let parent_hash = tip.hash(); + let mut child = tip.make_fake_child().set_block_commitment(commitment); + let too_early = super::any_ancestor_blocks(&non_finalized, &finalized.db, parent_hash) + .take(11) + .last() + .expect("the context has a median-time window") + .header + .time; + Arc::make_mut(&mut Arc::make_mut(&mut child).header).time = too_early; + let relevant_data = super::any_ancestor_blocks(&non_finalized, &finalized.db, parent_hash) + .take(POW_ADJUSTMENT_BLOCK_SPAN) + .map(|block| (block.header.difficulty_threshold, block.header.time)); + let expected_target = AdjustedDifficulty::new_from_block(&child, &network, relevant_data) + .expect("the context derives an expected target") + .expected_difficulty_threshold(); + Arc::make_mut(&mut Arc::make_mut(&mut child).header).difficulty_threshold = expected_target; + + let error = super::check_prepared_mined_relay_eligibility_for_state( + &network, + &non_finalized, + &finalized.db, + crate::BlockCommitmentData { + block: child, + auth_data_root: None, + }, + ) + .expect_err("a time at or below median-time-past fails the relay preflight"); + + assert!(matches!( + error.downcast_ref::(), + Some(ValidateContextError::TimeTooEarly { .. }) + )); +} + +#[test] +fn prepared_relay_preflight_rejects_a_forged_commitment() { + use crate::tests::FakeChainHelper; + + let _init_guard = zakura_test::init(); + let (network, finalized, non_finalized, side) = prepared_relay_test_state(); + let child = side.make_fake_child().set_block_commitment([0x42; 32]); + + let error = super::check_prepared_mined_relay_eligibility_for_state( + &network, + &non_finalized, + &finalized.db, + crate::BlockCommitmentData { + block: child, + auth_data_root: None, + }, + ) + .expect_err("a forged commitment fails the relay preflight"); + + assert!(matches!( + error.downcast_ref::(), + Some(ValidateContextError::InvalidBlockCommitment(_)) + )); +} + #[tokio::test] async fn historical_frontier_load_errors_are_returned_from_state_init() { let network = Network::Mainnet; diff --git a/docs/changelog/unreleased/748.md b/docs/changelog/unreleased/748.md index 3555c645d7..46fb39c5bc 100644 --- a/docs/changelog/unreleased/748.md +++ b/docs/changelog/unreleased/748.md @@ -1,4 +1,5 @@ ## Added -- Added early mined-block inventory after state admission. +- Added early inventory for prepared mined blocks after expected-work validation and state + admission. - Added prepared mining-candidate reuse through `workid`. From e5e4312f781e3f43b2c7f6b019f3b80570c5f17d Mon Sep 17 00:00:00 2001 From: evan-forbes Date: Sun, 23 Aug 2026 13:51:03 -0500 Subject: [PATCH 08/22] test(mining): allow safe committed inventory fallback --- crates/zakurad/tests/acceptance.rs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/crates/zakurad/tests/acceptance.rs b/crates/zakurad/tests/acceptance.rs index 4eba4ea774..7bdf9dac0d 100644 --- a/crates/zakurad/tests/acceptance.rs +++ b/crates/zakurad/tests/acceptance.rs @@ -3822,15 +3822,19 @@ async fn nu6_funding_streams_and_coinbase_balance() -> Result<()> { // Check that the submitblock channel received the submitted block let mut submit_block_receiver = submitblock_channel.receiver(); let submit_block_channel_data = submit_block_receiver.recv().await.expect("channel is open"); - assert!( - matches!( - submit_block_channel_data, - MinedBlockEvent::Early { hash, height, .. } - if hash == proposal_block.hash() - && height == proposal_block.coinbase_height().unwrap() + let (submitted_hash, submitted_height) = match submit_block_channel_data { + MinedBlockEvent::Early { hash, height, .. } + | MinedBlockEvent::Committed { + hash, + height, + early_advertised: false, + } => (hash, height), + event => panic!( + "submitblock should send an authorized early event or the safe committed fallback: {event:?}" ), - "submitblock channel should receive the early submitted-block event" - ); + }; + assert_eq!(submitted_hash, proposal_block.hash()); + assert_eq!(submitted_height, proposal_block.coinbase_height().unwrap()); // Use an invalid coinbase transaction (with an output value greater than the `block_subsidy + miner_fees - expected_lockbox_funding_stream`) From dbfc7fc51da6c0f5a694a867a3f27f81f19f7334 Mon Sep 17 00:00:00 2001 From: evan-forbes Date: Sat, 22 Aug 2026 13:48:52 -0500 Subject: [PATCH 09/22] fix(state): measure writer queue directly --- crates/zakura-state/src/service.rs | 4 +++- crates/zakura-state/src/service/write.rs | 19 +++++++++++++++---- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/crates/zakura-state/src/service.rs b/crates/zakura-state/src/service.rs index 8d854d2ab2..5f7e52f00b 100644 --- a/crates/zakura-state/src/service.rs +++ b/crates/zakura-state/src/service.rs @@ -1059,7 +1059,9 @@ impl StateService { let admission = queued_child.2.clone(); let send_result = non_finalized_block_write_sender.send(queued_child.into()); - if let Err(SendError(NonFinalizedWriteMessage::Commit(queued))) = send_result { + if let Err(SendError(NonFinalizedWriteMessage::Commit { queued, .. })) = + send_result + { // If Zebra is shutting down, drop blocks and return an error. Self::send_semantically_verified_block_error( queued, diff --git a/crates/zakura-state/src/service/write.rs b/crates/zakura-state/src/service/write.rs index 885e60c2af..e88ec4cfad 100644 --- a/crates/zakura-state/src/service/write.rs +++ b/crates/zakura-state/src/service/write.rs @@ -1546,7 +1546,12 @@ pub enum NonFinalizedWriteMessage { }, /// A newly downloaded and semantically verified block prepared for /// contextual validation and insertion into the non-finalized state. - Commit(QueuedSemanticallyVerified), + Commit { + /// The block, response channel, and optional lifecycle reporter. + queued: QueuedSemanticallyVerified, + /// The instant immediately before the state service attempted the channel send. + queued_at: Instant, + }, /// The hash of a block that should be invalidated and removed from /// the non-finalized state, if present. Invalidate { @@ -1563,7 +1568,10 @@ pub enum NonFinalizedWriteMessage { impl From for NonFinalizedWriteMessage { fn from(block: QueuedSemanticallyVerified) -> Self { - NonFinalizedWriteMessage::Commit(block) + NonFinalizedWriteMessage::Commit { + queued: block, + queued_at: Instant::now(), + } } } @@ -2437,7 +2445,7 @@ impl WriteBlockWorkerTask { let _ = rsp_tx.send(result); None } - NonFinalizedWriteMessage::Commit(queued_child) => Some(queued_child), + NonFinalizedWriteMessage::Commit { queued, queued_at } => Some((queued, queued_at)), NonFinalizedWriteMessage::Invalidate { hash, rsp_tx } => { tracing::info!(?hash, "invalidating a block in the non-finalized state"); let result = if let Some(writer) = header_chain.as_ref() { @@ -2502,10 +2510,13 @@ impl WriteBlockWorkerTask { } }; - let Some((queued_child, rsp_tx, _admission)) = queued_child_and_rsp_tx else { + let Some(((queued_child, rsp_tx, _admission), queued_at)) = queued_child_and_rsp_tx + else { continue; }; + metrics::histogram!("state.block_writer.queue.duration_seconds") + .record(queued_at.elapsed().as_secs_f64()); let child_hash = queued_child.hash; let parent_hash = queued_child.block.header.previous_block_hash; let child_height = queued_child.height; From 683d18609619ed0ce80c3f93ca46d1624079c99b Mon Sep 17 00:00:00 2001 From: evan-forbes Date: Sat, 22 Aug 2026 23:28:35 -0500 Subject: [PATCH 10/22] perf(state): measure mined block admission timing --- crates/zakura-consensus/src/block.rs | 1 + crates/zakura-state/src/request.rs | 3 +++ crates/zakura-state/src/service.rs | 15 ++++++++++++++- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/crates/zakura-consensus/src/block.rs b/crates/zakura-consensus/src/block.rs index e5a829b0be..b998fc0b18 100644 --- a/crates/zakura-consensus/src/block.rs +++ b/crates/zakura-consensus/src/block.rs @@ -571,6 +571,7 @@ where Some(admission) => zs::Request::CommitSemanticallyVerifiedBlockWithAdmission { block: prepared_block, admission, + requested_at: std::time::Instant::now(), }, None => zs::Request::CommitSemanticallyVerifiedBlock(prepared_block), }; diff --git a/crates/zakura-state/src/request.rs b/crates/zakura-state/src/request.rs index 548cdcdbe7..5ddca256f6 100644 --- a/crates/zakura-state/src/request.rs +++ b/crates/zakura-state/src/request.rs @@ -8,6 +8,7 @@ use std::{ atomic::{AtomicBool, AtomicU8, Ordering}, Arc, }, + time::Instant, }; use tokio::sync::Notify; @@ -1249,6 +1250,8 @@ pub enum Request { block: SemanticallyVerifiedBlock, /// The admission notification. admission: BlockAdmission, + /// When consensus submitted this request to the buffered state service. + requested_at: Instant, }, /// Commit a checkpointed block to the state, skipping most but not all diff --git a/crates/zakura-state/src/service.rs b/crates/zakura-state/src/service.rs index 5f7e52f00b..c7d2b64007 100644 --- a/crates/zakura-state/src/service.rs +++ b/crates/zakura-state/src/service.rs @@ -1561,16 +1561,29 @@ impl Service for StateService { .boxed() } - Request::CommitSemanticallyVerifiedBlockWithAdmission { block, admission } => { + Request::CommitSemanticallyVerifiedBlockWithAdmission { + block, + admission, + requested_at, + } => { let timer = CodeTimer::start(); + metrics::histogram!("state.semantic_commit.dispatch.duration_seconds") + .record(requested_at.elapsed().as_secs_f64()); + + let prequeue_checks_start = Instant::now(); self.assert_block_can_be_validated(&block); self.pending_utxos.check_against_ordered(&block.new_outputs); + metrics::histogram!("state.semantic_commit.prequeue_checks.duration_seconds") + .record(prequeue_checks_start.elapsed().as_secs_f64()); + let queue_send_start = Instant::now(); let rsp_rx = tokio::task::block_in_place(move || { span.in_scope(|| { self.queue_and_commit_to_non_finalized_state(block, Some(admission)) }) }); + metrics::histogram!("state.semantic_commit.queue_send.duration_seconds") + .record(queue_send_start.elapsed().as_secs_f64()); timer.finish_desc("CommitSemanticallyVerifiedBlockWithAdmission"); let span = Span::current(); From e112370a4d6df297379e9aecbfdfa6090f4c2023 Mon Sep 17 00:00:00 2001 From: evan-forbes Date: Sun, 23 Aug 2026 00:52:26 -0500 Subject: [PATCH 11/22] fix(state): import writer timing clock --- crates/zakura-state/src/service/write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/zakura-state/src/service/write.rs b/crates/zakura-state/src/service/write.rs index e88ec4cfad..2f66d5dc85 100644 --- a/crates/zakura-state/src/service/write.rs +++ b/crates/zakura-state/src/service/write.rs @@ -5,7 +5,7 @@ use std::{ panic::{catch_unwind, resume_unwind, AssertUnwindSafe}, path::{Path, PathBuf}, sync::{Arc, OnceLock}, - time::Duration, + time::{Duration, Instant}, }; use indexmap::IndexMap; From bdf35bb750848eab61e5250d12ea2b4cf06f451a Mon Sep 17 00:00:00 2001 From: evan-forbes Date: Sun, 23 Aug 2026 02:39:14 -0500 Subject: [PATCH 12/22] fix(state): make admission metrics composable --- crates/zakura-consensus/src/block.rs | 19 ++++++++++++------- crates/zakura-state/src/service.rs | 2 +- crates/zakura-state/src/service/write.rs | 9 +++++++-- docs/changelog/unreleased/781.md | 4 ++++ 4 files changed, 24 insertions(+), 10 deletions(-) create mode 100644 docs/changelog/unreleased/781.md diff --git a/crates/zakura-consensus/src/block.rs b/crates/zakura-consensus/src/block.rs index b998fc0b18..a007e00f24 100644 --- a/crates/zakura-consensus/src/block.rs +++ b/crates/zakura-consensus/src/block.rs @@ -567,6 +567,17 @@ where { let hash = prepared_block.hash; let is_mined_commit = admission.is_some(); + let commit_start = std::time::Instant::now(); + let ready_start = std::time::Instant::now(); + let ready_state_service = state_service + .ready() + .await + .map_err(|source| VerifyBlockError::StateService { source, hash })?; + if is_mined_commit { + metrics::histogram!("state.semantic_commit.ready_wait.duration_seconds") + .record(ready_start.elapsed().as_secs_f64()); + } + let request = match admission { Some(admission) => zs::Request::CommitSemanticallyVerifiedBlockWithAdmission { block: prepared_block, @@ -575,13 +586,7 @@ where }, None => zs::Request::CommitSemanticallyVerifiedBlock(prepared_block), }; - let commit_start = std::time::Instant::now(); - let response = state_service - .ready() - .await - .map_err(|source| VerifyBlockError::StateService { source, hash })? - .call(request) - .await; + let response = ready_state_service.call(request).await; if is_mined_commit { metrics::histogram!("mining.contextual_commit.duration_seconds") .record(commit_start.elapsed().as_secs_f64()); diff --git a/crates/zakura-state/src/service.rs b/crates/zakura-state/src/service.rs index c7d2b64007..ce988c01a3 100644 --- a/crates/zakura-state/src/service.rs +++ b/crates/zakura-state/src/service.rs @@ -1582,7 +1582,7 @@ impl Service for StateService { self.queue_and_commit_to_non_finalized_state(block, Some(admission)) }) }); - metrics::histogram!("state.semantic_commit.queue_send.duration_seconds") + metrics::histogram!("state.semantic_commit.queue_and_commit.duration_seconds") .record(queue_send_start.elapsed().as_secs_f64()); timer.finish_desc("CommitSemanticallyVerifiedBlockWithAdmission"); diff --git a/crates/zakura-state/src/service/write.rs b/crates/zakura-state/src/service/write.rs index 2f66d5dc85..1cccd2bfb9 100644 --- a/crates/zakura-state/src/service/write.rs +++ b/crates/zakura-state/src/service/write.rs @@ -2510,13 +2510,18 @@ impl WriteBlockWorkerTask { } }; - let Some(((queued_child, rsp_tx, _admission), queued_at)) = queued_child_and_rsp_tx + let Some(((queued_child, rsp_tx, admission), queued_at)) = queued_child_and_rsp_tx else { continue; }; + let writer_queue_duration = queued_at.elapsed().as_secs_f64(); metrics::histogram!("state.block_writer.queue.duration_seconds") - .record(queued_at.elapsed().as_secs_f64()); + .record(writer_queue_duration); + if admission.is_some() { + metrics::histogram!("state.block_writer.queue.mined.duration_seconds") + .record(writer_queue_duration); + } let child_hash = queued_child.hash; let parent_hash = queued_child.block.header.previous_block_hash; let child_height = queued_child.height; diff --git a/docs/changelog/unreleased/781.md b/docs/changelog/unreleased/781.md new file mode 100644 index 0000000000..4e0b214256 --- /dev/null +++ b/docs/changelog/unreleased/781.md @@ -0,0 +1,4 @@ + + +This PR adds internal mined-block admission metrics and has no operator- or +crate-consumer-visible effect. From 5d7473f60b7a0d481bfefe4ba8c1a100eeb3a268 Mon Sep 17 00:00:00 2001 From: evan-forbes Date: Sat, 22 Aug 2026 14:29:52 -0500 Subject: [PATCH 13/22] perf(state): measure contextual phases --- .../src/service/non_finalized_state.rs | 87 ++++++++++++++----- crates/zakura-state/src/service/write.rs | 30 +++++-- 2 files changed, 86 insertions(+), 31 deletions(-) diff --git a/crates/zakura-state/src/service/non_finalized_state.rs b/crates/zakura-state/src/service/non_finalized_state.rs index bdc4b900de..c309fb7510 100644 --- a/crates/zakura-state/src/service/non_finalized_state.rs +++ b/crates/zakura-state/src/service/non_finalized_state.rs @@ -7,6 +7,7 @@ use std::{ mem, path::{Path, PathBuf}, sync::Arc, + time::Instant, }; use indexmap::IndexMap; @@ -579,28 +580,41 @@ impl NonFinalizedState { // Reads from disk // // TODO: if these disk reads show up in profiles, run them in parallel, using std::thread::spawn() + let transparent_spend_start = Instant::now(); let spent_utxos = check::utxo::transparent_spend( &prepared, &new_chain.unspent_utxos(), &new_chain.spent_utxos, finalized_state, - )?; + ); + metrics::histogram!("state.contextual.transparent_spend.duration_seconds") + .record(transparent_spend_start.elapsed().as_secs_f64()); + let spent_utxos = spent_utxos?; // Reads from disk - check::anchors::block_sapling_orchard_ironwood_anchors_refer_to_final_treestates( - finalized_state, - &new_chain, - &prepared, - )?; + let shielded_anchor_start = Instant::now(); + let shielded_anchors = + check::anchors::block_sapling_orchard_ironwood_anchors_refer_to_final_treestates( + finalized_state, + &new_chain, + &prepared, + ); + metrics::histogram!("state.contextual.shielded_anchors.duration_seconds") + .record(shielded_anchor_start.elapsed().as_secs_f64()); + shielded_anchors?; // Reads from disk + let sprout_anchor_fetch_start = Instant::now(); let sprout_final_treestates = check::anchors::block_fetch_sprout_final_treestates( finalized_state, &new_chain, &prepared, ); + metrics::histogram!("state.contextual.sprout_anchor_fetch.duration_seconds") + .record(sprout_anchor_fetch_start.elapsed().as_secs_f64()); // Quick check that doesn't read from disk + let contextual_block_start = Instant::now(); let contextual = ContextuallyVerifiedBlock::with_block_and_spent_utxos( prepared.clone(), spent_utxos.clone(), @@ -613,9 +627,17 @@ impl NonFinalizedState { transaction_count: prepared.block.transactions.len(), spent_utxo_count: spent_utxos.len(), } - })?; - - Self::validate_and_update_parallel(new_chain, contextual, sprout_final_treestates) + }); + metrics::histogram!("state.contextual.block_construction.duration_seconds") + .record(contextual_block_start.elapsed().as_secs_f64()); + let contextual = contextual?; + + let parallel_update_start = Instant::now(); + let result = + Self::validate_and_update_parallel(new_chain, contextual, sprout_final_treestates); + metrics::histogram!("state.contextual.parallel_update.duration_seconds") + .record(parallel_update_start.elapsed().as_secs_f64()); + result } /// Validate `contextual` and update `new_chain`, doing CPU-intensive work in parallel batches. @@ -641,22 +663,25 @@ impl NonFinalizedState { rayon::in_place_scope_fifo(|scope| { scope.spawn_fifo(|_scope| { - block_commitment_result = Some(check::block_commitment_is_valid_for_chain_history( + let start = Instant::now(); + let result = check::block_commitment_is_valid_for_chain_history( block, &network, &history_tree, None, - )); + ); + block_commitment_result = Some((result, start.elapsed())); }); scope.spawn_fifo(|_scope| { - sprout_anchor_result = - Some(check::anchors::block_sprout_anchors_refer_to_treestates( - sprout_final_treestates, - block2, - transaction_hashes, - height, - )); + let start = Instant::now(); + let result = check::anchors::block_sprout_anchors_refer_to_treestates( + sprout_final_treestates, + block2, + transaction_hashes, + height, + ); + sprout_anchor_result = Some((result, start.elapsed())); }); // We're pretty sure the new block is valid, @@ -665,19 +690,35 @@ impl NonFinalizedState { // Pushing a block onto a Chain can launch additional parallel batches. // TODO: should we pass _scope into Chain::push()? scope.spawn_fifo(|_scope| { + let start = Instant::now(); // TODO: Replace with Arc::unwrap_or_clone() when it stabilises: // https://github.com/rust-lang/rust/issues/93610 let new_chain = Arc::try_unwrap(new_chain) .unwrap_or_else(|shared_chain| (*shared_chain).clone()); - chain_push_result = Some(new_chain.push(contextual).map(Arc::new)); + let result = new_chain.push(contextual).map(Arc::new); + chain_push_result = Some((result, start.elapsed())); }); }); // Don't return the updated Chain unless all the parallel results were Ok - block_commitment_result.expect("scope has finished")?; - sprout_anchor_result.expect("scope has finished")?; - - chain_push_result.expect("scope has finished") + let (block_commitment_result, block_commitment_duration) = + block_commitment_result.expect("scope has finished"); + let (sprout_anchor_result, sprout_anchor_duration) = + sprout_anchor_result.expect("scope has finished"); + let (chain_push_result, chain_push_duration) = + chain_push_result.expect("scope has finished"); + + metrics::histogram!("state.contextual.block_commitment.duration_seconds") + .record(block_commitment_duration.as_secs_f64()); + metrics::histogram!("state.contextual.sprout_anchor_check.duration_seconds") + .record(sprout_anchor_duration.as_secs_f64()); + metrics::histogram!("state.contextual.chain_push.duration_seconds") + .record(chain_push_duration.as_secs_f64()); + + block_commitment_result?; + sprout_anchor_result?; + + chain_push_result } /// Returns the length of the non-finalized portion of the current best chain diff --git a/crates/zakura-state/src/service/write.rs b/crates/zakura-state/src/service/write.rs index 1cccd2bfb9..8f0ed21f79 100644 --- a/crates/zakura-state/src/service/write.rs +++ b/crates/zakura-state/src/service/write.rs @@ -1276,7 +1276,12 @@ pub(crate) fn validate_and_commit_non_finalized( non_finalized_state: &mut NonFinalizedState, prepared: SemanticallyVerifiedBlock, ) -> Result<(), ValidateContextError> { - check::initial_contextual_validity(finalized_state, non_finalized_state, &prepared)?; + let initial_checks_start = Instant::now(); + let initial_checks = + check::initial_contextual_validity(finalized_state, non_finalized_state, &prepared); + metrics::histogram!("state.contextual.initial_checks.duration_seconds") + .record(initial_checks_start.elapsed().as_secs_f64()); + initial_checks?; let parent_hash = prepared.block.header.previous_block_hash; if finalized_state.finalized_tip_hash() == parent_hash { @@ -2538,7 +2543,10 @@ impl WriteBlockWorkerTask { } else { tracing::trace!(?child_hash, "validating queued child"); if let Some(writer) = header_chain.as_ref() { + let snapshot_clone_start = Instant::now(); let mut staged = non_finalized_state.clone(); + metrics::histogram!("state.contextual.snapshot_clone.duration_seconds") + .record(snapshot_clone_start.elapsed().as_secs_f64()); validate_and_commit_non_finalized( &finalized_state.db, &mut staged, @@ -2547,12 +2555,13 @@ impl WriteBlockWorkerTask { .map_err(|error| CommitBlockError::from(Box::new(error))) .and_then(|()| { let accepted = Frontier::new(child_height, child_hash); + let transition_prepare_start = Instant::now(); let (evidence, event_path, request) = verified_request(writer, non_finalized_state, &staged, accepted) .map_err(|error| CommitBlockError::HeaderChainError { error: error.to_string(), })?; - PreparedFullStateTransition::new( + let transition = PreparedFullStateTransition::new( evidence, writer .runtime @@ -2565,16 +2574,21 @@ impl WriteBlockWorkerTask { None, request, ) - .map_err(|error| CommitBlockError::HeaderChainError { - error: error.to_string(), - })? - .commit(&writer.runtime, non_finalized_state, &writer.context()) - .map(|_| ()) .map_err(|error| { CommitBlockError::HeaderChainError { error: error.to_string(), } - }) + })?; + metrics::histogram!( + "state.contextual.header_transition_prepare.duration_seconds" + ) + .record(transition_prepare_start.elapsed().as_secs_f64()); + transition + .commit(&writer.runtime, non_finalized_state, &writer.context()) + .map(|_| ()) + .map_err(|error| CommitBlockError::HeaderChainError { + error: error.to_string(), + }) }) } else { validate_and_commit_non_finalized( From abc601bf82eb278ee887d5fb1d602e5b5049eb9f Mon Sep 17 00:00:00 2001 From: evan-forbes Date: Sun, 23 Aug 2026 03:03:06 -0500 Subject: [PATCH 14/22] fix(state): make contextual phase metrics actionable --- .../src/service/non_finalized_state.rs | 211 ++++++++++++++---- crates/zakura-state/src/service/write.rs | 142 ++++++++---- docs/changelog/unreleased/782.md | 4 + 3 files changed, 267 insertions(+), 90 deletions(-) create mode 100644 docs/changelog/unreleased/782.md diff --git a/crates/zakura-state/src/service/non_finalized_state.rs b/crates/zakura-state/src/service/non_finalized_state.rs index c309fb7510..3507419284 100644 --- a/crates/zakura-state/src/service/non_finalized_state.rs +++ b/crates/zakura-state/src/service/non_finalized_state.rs @@ -39,6 +39,16 @@ mod tests; pub(crate) use backup::write_semantically_verified_backup_block; pub(crate) use chain::{Chain, SpendingTransactionId}; +macro_rules! record_contextual_duration { + ($metric_name:literal, $mined_metric_name:literal, $duration:expr, $is_mined:expr $(,)?) => {{ + let duration = $duration.as_secs_f64(); + metrics::histogram!($metric_name).record(duration); + if $is_mined { + metrics::histogram!($mined_metric_name).record(duration); + } + }}; +} + /// The state of the chains in memory, including queued blocks. /// /// Clones of the non-finalized state contain independent copies of the chains. @@ -351,20 +361,42 @@ impl NonFinalizedState { /// Commit block to the non-finalized state, on top of: /// - an existing chain's tip, or /// - a newly forked chain. - #[tracing::instrument(level = "debug", skip(self, finalized_state, prepared))] pub fn commit_block( &mut self, prepared: SemanticallyVerifiedBlock, finalized_state: &ZakuraDb, + ) -> Result<(), ValidateContextError> { + self.commit_block_with_metrics(prepared, finalized_state, false) + } + + #[tracing::instrument( + name = "commit_block", + level = "debug", + skip(self, finalized_state, prepared) + )] + pub(crate) fn commit_block_with_metrics( + &mut self, + prepared: SemanticallyVerifiedBlock, + finalized_state: &ZakuraDb, + is_mined: bool, ) -> Result<(), ValidateContextError> { let parent_hash = prepared.block.header.previous_block_hash; let (height, hash) = (prepared.height, prepared.hash); - let parent_chain = self.parent_chain(parent_hash)?; + let parent_chain_start = Instant::now(); + let parent_chain = self.parent_chain(parent_hash); + record_contextual_duration!( + "state.contextual.parent_chain.duration_seconds", + "state.contextual.mined.parent_chain.duration_seconds", + parent_chain_start.elapsed(), + is_mined, + ); + let parent_chain = parent_chain?; // If the block is invalid, return the error, // and drop the cloned parent Arc, or newly created chain fork. - let modified_chain = self.validate_and_commit(parent_chain, prepared, finalized_state)?; + let modified_chain = + self.validate_and_commit(parent_chain, prepared, finalized_state, is_mined)?; // If the block is valid: // - add the new chain fork or updated chain to the set of recent chains @@ -517,36 +549,63 @@ impl NonFinalizedState { /// Commit block to the non-finalized state as a new chain where its parent /// is the finalized tip. - #[tracing::instrument(level = "debug", skip(self, finalized_state, prepared))] #[allow(clippy::unwrap_in_result)] pub fn commit_new_chain( &mut self, prepared: SemanticallyVerifiedBlock, finalized_state: &ZakuraDb, ) -> Result<(), ValidateContextError> { - let finalized_tip_height = finalized_state.finalized_tip_height(); - - // TODO: fix tests that don't initialize the finalized state - #[cfg(not(test))] - let finalized_tip_height = finalized_tip_height.expect("finalized state contains blocks"); - #[cfg(test)] - let finalized_tip_height = finalized_tip_height.unwrap_or(zakura_chain::block::Height(0)); - - let chain = Chain::new( - &self.network, - finalized_tip_height, - finalized_state.sprout_tree_for_tip()?, - finalized_state.sapling_tree_for_tip(), - finalized_state.orchard_tree_for_tip(), - finalized_state.ironwood_tree_for_tip(), - finalized_state.history_tree(), - finalized_state.finalized_value_pool(), + self.commit_new_chain_with_metrics(prepared, finalized_state, false) + } + + #[tracing::instrument( + name = "commit_new_chain", + level = "debug", + skip(self, finalized_state, prepared) + )] + #[allow(clippy::unwrap_in_result)] + pub(crate) fn commit_new_chain_with_metrics( + &mut self, + prepared: SemanticallyVerifiedBlock, + finalized_state: &ZakuraDb, + is_mined: bool, + ) -> Result<(), ValidateContextError> { + let chain_new_start = Instant::now(); + let chain: Result = (|| { + let finalized_tip_height = finalized_state.finalized_tip_height(); + + // TODO: fix tests that don't initialize the finalized state + #[cfg(not(test))] + let finalized_tip_height = + finalized_tip_height.expect("finalized state contains blocks"); + #[cfg(test)] + let finalized_tip_height = + finalized_tip_height.unwrap_or(zakura_chain::block::Height(0)); + + Ok(Chain::new( + &self.network, + finalized_tip_height, + finalized_state.sprout_tree_for_tip()?, + finalized_state.sapling_tree_for_tip(), + finalized_state.orchard_tree_for_tip(), + finalized_state.ironwood_tree_for_tip(), + finalized_state.history_tree(), + finalized_state.finalized_value_pool(), + )) + })(); + record_contextual_duration!( + "state.contextual.chain_new.duration_seconds", + "state.contextual.mined.chain_new.duration_seconds", + chain_new_start.elapsed(), + is_mined, ); + let chain = chain?; let (height, hash) = (prepared.height, prepared.hash); // If the block is invalid, return the error, and drop the newly created chain fork - let chain = self.validate_and_commit(Arc::new(chain), prepared, finalized_state)?; + let chain = + self.validate_and_commit(Arc::new(chain), prepared, finalized_state, is_mined)?; // If the block is valid, add the new chain fork to the set of recent chains. self.insert(chain); @@ -566,6 +625,7 @@ impl NonFinalizedState { new_chain: Arc, prepared: SemanticallyVerifiedBlock, finalized_state: &ZakuraDb, + is_mined: bool, ) -> Result, ValidateContextError> { if self .invalidated_blocks @@ -580,15 +640,28 @@ impl NonFinalizedState { // Reads from disk // // TODO: if these disk reads show up in profiles, run them in parallel, using std::thread::spawn() + let unspent_utxo_snapshot_start = Instant::now(); + let unspent_utxos = new_chain.unspent_utxos(); + record_contextual_duration!( + "state.contextual.unspent_utxo_snapshot.duration_seconds", + "state.contextual.mined.unspent_utxo_snapshot.duration_seconds", + unspent_utxo_snapshot_start.elapsed(), + is_mined, + ); + let transparent_spend_start = Instant::now(); let spent_utxos = check::utxo::transparent_spend( &prepared, - &new_chain.unspent_utxos(), + &unspent_utxos, &new_chain.spent_utxos, finalized_state, ); - metrics::histogram!("state.contextual.transparent_spend.duration_seconds") - .record(transparent_spend_start.elapsed().as_secs_f64()); + record_contextual_duration!( + "state.contextual.transparent_spend.duration_seconds", + "state.contextual.mined.transparent_spend.duration_seconds", + transparent_spend_start.elapsed(), + is_mined, + ); let spent_utxos = spent_utxos?; // Reads from disk @@ -599,8 +672,12 @@ impl NonFinalizedState { &new_chain, &prepared, ); - metrics::histogram!("state.contextual.shielded_anchors.duration_seconds") - .record(shielded_anchor_start.elapsed().as_secs_f64()); + record_contextual_duration!( + "state.contextual.shielded_anchors.duration_seconds", + "state.contextual.mined.shielded_anchors.duration_seconds", + shielded_anchor_start.elapsed(), + is_mined, + ); shielded_anchors?; // Reads from disk @@ -610,8 +687,12 @@ impl NonFinalizedState { &new_chain, &prepared, ); - metrics::histogram!("state.contextual.sprout_anchor_fetch.duration_seconds") - .record(sprout_anchor_fetch_start.elapsed().as_secs_f64()); + record_contextual_duration!( + "state.contextual.sprout_anchor_fetch.duration_seconds", + "state.contextual.mined.sprout_anchor_fetch.duration_seconds", + sprout_anchor_fetch_start.elapsed(), + is_mined, + ); // Quick check that doesn't read from disk let contextual_block_start = Instant::now(); @@ -628,15 +709,27 @@ impl NonFinalizedState { spent_utxo_count: spent_utxos.len(), } }); - metrics::histogram!("state.contextual.block_construction.duration_seconds") - .record(contextual_block_start.elapsed().as_secs_f64()); + record_contextual_duration!( + "state.contextual.block_construction.duration_seconds", + "state.contextual.mined.block_construction.duration_seconds", + contextual_block_start.elapsed(), + is_mined, + ); let contextual = contextual?; let parallel_update_start = Instant::now(); - let result = - Self::validate_and_update_parallel(new_chain, contextual, sprout_final_treestates); - metrics::histogram!("state.contextual.parallel_update.duration_seconds") - .record(parallel_update_start.elapsed().as_secs_f64()); + let result = Self::validate_and_update_parallel( + new_chain, + contextual, + sprout_final_treestates, + is_mined, + ); + record_contextual_duration!( + "state.contextual.parallel_update.duration_seconds", + "state.contextual.mined.parallel_update.duration_seconds", + parallel_update_start.elapsed(), + is_mined, + ); result } @@ -647,6 +740,7 @@ impl NonFinalizedState { new_chain: Arc, contextual: ContextuallyVerifiedBlock, sprout_final_treestates: HashMap>, + is_mined: bool, ) -> Result, ValidateContextError> { let mut block_commitment_result = None; let mut sprout_anchor_result = None; @@ -690,13 +784,14 @@ impl NonFinalizedState { // Pushing a block onto a Chain can launch additional parallel batches. // TODO: should we pass _scope into Chain::push()? scope.spawn_fifo(|_scope| { - let start = Instant::now(); - // TODO: Replace with Arc::unwrap_or_clone() when it stabilises: - // https://github.com/rust-lang/rust/issues/93610 - let new_chain = Arc::try_unwrap(new_chain) - .unwrap_or_else(|shared_chain| (*shared_chain).clone()); + let chain_clone_start = Instant::now(); + let new_chain = Arc::unwrap_or_clone(new_chain); + let chain_clone_duration = chain_clone_start.elapsed(); + + let chain_push_start = Instant::now(); let result = new_chain.push(contextual).map(Arc::new); - chain_push_result = Some((result, start.elapsed())); + chain_push_result = + Some((result, chain_clone_duration, chain_push_start.elapsed())); }); }); @@ -705,15 +800,35 @@ impl NonFinalizedState { block_commitment_result.expect("scope has finished"); let (sprout_anchor_result, sprout_anchor_duration) = sprout_anchor_result.expect("scope has finished"); - let (chain_push_result, chain_push_duration) = + let (chain_push_result, chain_clone_duration, chain_push_duration) = chain_push_result.expect("scope has finished"); - metrics::histogram!("state.contextual.block_commitment.duration_seconds") - .record(block_commitment_duration.as_secs_f64()); - metrics::histogram!("state.contextual.sprout_anchor_check.duration_seconds") - .record(sprout_anchor_duration.as_secs_f64()); - metrics::histogram!("state.contextual.chain_push.duration_seconds") - .record(chain_push_duration.as_secs_f64()); + // These task durations overlap. Only `parallel_update` measures their + // combined critical-path wall time. + record_contextual_duration!( + "state.contextual.parallel_task.block_commitment.duration_seconds", + "state.contextual.mined.parallel_task.block_commitment.duration_seconds", + block_commitment_duration, + is_mined, + ); + record_contextual_duration!( + "state.contextual.parallel_task.sprout_anchor_check.duration_seconds", + "state.contextual.mined.parallel_task.sprout_anchor_check.duration_seconds", + sprout_anchor_duration, + is_mined, + ); + record_contextual_duration!( + "state.contextual.parallel_task.chain_clone.duration_seconds", + "state.contextual.mined.parallel_task.chain_clone.duration_seconds", + chain_clone_duration, + is_mined, + ); + record_contextual_duration!( + "state.contextual.parallel_task.chain_push.duration_seconds", + "state.contextual.mined.parallel_task.chain_push.duration_seconds", + chain_push_duration, + is_mined, + ); block_commitment_result?; sprout_anchor_result?; diff --git a/crates/zakura-state/src/service/write.rs b/crates/zakura-state/src/service/write.rs index 8f0ed21f79..32ef5a9380 100644 --- a/crates/zakura-state/src/service/write.rs +++ b/crates/zakura-state/src/service/write.rs @@ -1260,9 +1260,33 @@ fn commit_operator_change( /// We allow enough space for multiple concurrent chain forks with errors. const REJECTED_ANCESTOR_MAP_LIMIT: usize = MAX_BLOCK_REORG_HEIGHT as usize * 2; +macro_rules! record_contextual_duration { + ($metric_name:literal, $mined_metric_name:literal, $duration:expr, $is_mined:expr $(,)?) => {{ + let duration = $duration.as_secs_f64(); + metrics::histogram!($metric_name).record(duration); + if $is_mined { + metrics::histogram!($mined_metric_name).record(duration); + } + }}; +} + /// Run contextual validation on the prepared block and add it to the /// non-finalized state if it is contextually valid. +pub(crate) fn validate_and_commit_non_finalized( + finalized_state: &ZakuraDb, + non_finalized_state: &mut NonFinalizedState, + prepared: SemanticallyVerifiedBlock, +) -> Result<(), ValidateContextError> { + validate_and_commit_non_finalized_with_metrics( + finalized_state, + non_finalized_state, + prepared, + false, + ) +} + #[tracing::instrument( + name = "validate_and_commit_non_finalized", level = "debug", skip(finalized_state, non_finalized_state, prepared), fields( @@ -1271,26 +1295,39 @@ const REJECTED_ANCESTOR_MAP_LIMIT: usize = MAX_BLOCK_REORG_HEIGHT as usize * 2; chains = non_finalized_state.chain_count() ) )] -pub(crate) fn validate_and_commit_non_finalized( +fn validate_and_commit_non_finalized_with_metrics( finalized_state: &ZakuraDb, non_finalized_state: &mut NonFinalizedState, prepared: SemanticallyVerifiedBlock, + is_mined: bool, ) -> Result<(), ValidateContextError> { + let total_start = Instant::now(); let initial_checks_start = Instant::now(); let initial_checks = check::initial_contextual_validity(finalized_state, non_finalized_state, &prepared); - metrics::histogram!("state.contextual.initial_checks.duration_seconds") - .record(initial_checks_start.elapsed().as_secs_f64()); - initial_checks?; - let parent_hash = prepared.block.header.previous_block_hash; + record_contextual_duration!( + "state.contextual.initial_checks.duration_seconds", + "state.contextual.mined.initial_checks.duration_seconds", + initial_checks_start.elapsed(), + is_mined, + ); + let result = initial_checks.and_then(|()| { + let parent_hash = prepared.block.header.previous_block_hash; - if finalized_state.finalized_tip_hash() == parent_hash { - non_finalized_state.commit_new_chain(prepared, finalized_state)?; - } else { - non_finalized_state.commit_block(prepared, finalized_state)?; - } + if finalized_state.finalized_tip_hash() == parent_hash { + non_finalized_state.commit_new_chain_with_metrics(prepared, finalized_state, is_mined) + } else { + non_finalized_state.commit_block_with_metrics(prepared, finalized_state, is_mined) + } + }); + record_contextual_duration!( + "state.contextual.total.duration_seconds", + "state.contextual.mined.total.duration_seconds", + total_start.elapsed(), + is_mined, + ); - Ok(()) + result } /// Update the [`LatestChainTip`], [`ChainTipChange`], and `non_finalized_state_sender` @@ -2523,7 +2560,8 @@ impl WriteBlockWorkerTask { let writer_queue_duration = queued_at.elapsed().as_secs_f64(); metrics::histogram!("state.block_writer.queue.duration_seconds") .record(writer_queue_duration); - if admission.is_some() { + let is_mined = admission.is_some(); + if is_mined { metrics::histogram!("state.block_writer.queue.mined.duration_seconds") .record(writer_queue_duration); } @@ -2545,56 +2583,76 @@ impl WriteBlockWorkerTask { if let Some(writer) = header_chain.as_ref() { let snapshot_clone_start = Instant::now(); let mut staged = non_finalized_state.clone(); - metrics::histogram!("state.contextual.snapshot_clone.duration_seconds") - .record(snapshot_clone_start.elapsed().as_secs_f64()); - validate_and_commit_non_finalized( + record_contextual_duration!( + "state.contextual.snapshot_clone.duration_seconds", + "state.contextual.mined.snapshot_clone.duration_seconds", + snapshot_clone_start.elapsed(), + is_mined, + ); + validate_and_commit_non_finalized_with_metrics( &finalized_state.db, &mut staged, queued_child, + is_mined, ) .map_err(|error| CommitBlockError::from(Box::new(error))) .and_then(|()| { let accepted = Frontier::new(child_height, child_hash); let transition_prepare_start = Instant::now(); - let (evidence, event_path, request) = + let transition = verified_request(writer, non_finalized_state, &staged, accepted) .map_err(|error| CommitBlockError::HeaderChainError { error: error.to_string(), - })?; - let transition = PreparedFullStateTransition::new( - evidence, - writer - .runtime - .publisher() - .snapshot() - .frontiers - .verified_best, - event_path, - staged, - None, - request, - ) - .map_err(|error| { - CommitBlockError::HeaderChainError { - error: error.to_string(), - } - })?; - metrics::histogram!( - "state.contextual.header_transition_prepare.duration_seconds" - ) - .record(transition_prepare_start.elapsed().as_secs_f64()); - transition + }) + .and_then(|(evidence, event_path, request)| { + PreparedFullStateTransition::new( + evidence, + writer + .runtime + .publisher() + .snapshot() + .frontiers + .verified_best, + event_path, + staged, + None, + request, + ) + .map_err(|error| { + CommitBlockError::HeaderChainError { + error: error.to_string(), + } + }) + }); + record_contextual_duration!( + "state.contextual.header_transition_prepare.duration_seconds", + "state.contextual.mined.header_transition_prepare.duration_seconds", + transition_prepare_start.elapsed(), + is_mined, + ); + let transition = transition?; + + let transition_commit_start = Instant::now(); + let result = transition .commit(&writer.runtime, non_finalized_state, &writer.context()) .map(|_| ()) .map_err(|error| CommitBlockError::HeaderChainError { error: error.to_string(), - }) + }); + record_contextual_duration!( + "state.contextual.header_transition_commit.duration_seconds", + "state.contextual.mined.header_transition_commit.duration_seconds", + transition_commit_start.elapsed(), + is_mined, + ); + result }) } else { - validate_and_commit_non_finalized( + validate_and_commit_non_finalized_with_metrics( &finalized_state.db, non_finalized_state, queued_child, + is_mined, ) .map_err(|error| CommitBlockError::from(Box::new(error))) } diff --git a/docs/changelog/unreleased/782.md b/docs/changelog/unreleased/782.md new file mode 100644 index 0000000000..b3c92f6619 --- /dev/null +++ b/docs/changelog/unreleased/782.md @@ -0,0 +1,4 @@ + + +This PR adds internal contextual-verification metrics and has no operator- or +crate-consumer-visible effect. From ee3bf0a8f7d7b10c4886bba1625f86125ff48649 Mon Sep 17 00:00:00 2001 From: evan-forbes Date: Mon, 24 Aug 2026 14:25:17 -0500 Subject: [PATCH 15/22] fix(mining): address final review hardening --- crates/zakura-consensus/src/block.rs | 9 +- crates/zakura-consensus/src/block/prepared.rs | 21 +- crates/zakura-consensus/src/block/tests.rs | 235 +++++++++++++++++- crates/zakura-rpc/src/methods.rs | 108 ++++---- .../src/methods/types/submit_block.rs | 137 +++++++--- crates/zakura-state/src/service.rs | 11 +- crates/zakura-state/src/service/tests.rs | 45 +++- crates/zakurad/src/components/inbound.rs | 70 ++++-- .../zakurad/src/components/inbound/tests.rs | 6 +- docs/changelog/params.md | 10 + docs/changelog/unreleased/748.md | 7 +- 11 files changed, 531 insertions(+), 128 deletions(-) diff --git a/crates/zakura-consensus/src/block.rs b/crates/zakura-consensus/src/block.rs index e5a829b0be..b89ff99883 100644 --- a/crates/zakura-consensus/src/block.rs +++ b/crates/zakura-consensus/src/block.rs @@ -323,6 +323,11 @@ where tx::check::lock_time_has_passed(transaction, height, block.header.time) .map_err(VerifyBlockError::Transaction)?; } + check::merkle_root_validity( + &network, + &block, + &prepared_block.transaction_hashes, + )?; metrics::histogram!("mining.solved_header_check.duration_seconds") .record(solved_header_start.elapsed().as_secs_f64()); @@ -495,7 +500,7 @@ where // Return early for proposal requests. if request.is_proposal() { - let cache_copy = prepared_block.clone(); + let cache_copy = request.should_cache().then(|| prepared_block.clone()); let response = match state_service .ready() .await @@ -507,7 +512,7 @@ where zs::Response::ValidBlockProposal => Ok(hash), _ => unreachable!("wrong response for CheckBlockProposalValidity"), }; - if response.is_ok() && request.should_cache() { + if let (Ok(_), Some(cache_copy)) = (&response, cache_copy) { let candidate = cache_copy.block.clone(); prepared_candidates.insert(&candidate, request.work_id(), cache_copy, &network); metrics::histogram!("mining.preparation.duration_seconds").record( diff --git a/crates/zakura-consensus/src/block/prepared.rs b/crates/zakura-consensus/src/block/prepared.rs index 371ca2c0db..74d0d19f30 100644 --- a/crates/zakura-consensus/src/block/prepared.rs +++ b/crates/zakura-consensus/src/block/prepared.rs @@ -106,6 +106,15 @@ impl PreparedCandidateCache { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); inner.prune_expired(); + if work_id.is_some_and(|work_id| { + inner.entries.iter().any(|entry| { + entry.work_id.as_deref() == Some(work_id) + && entry.immutable_bytes != immutable_bytes + }) + }) { + metrics::counter!("mining.prepared_cache.work_id_conflicts").increment(1); + return; + } let existing_work_id = if work_id.is_none() { inner .entries @@ -273,6 +282,12 @@ mod tests { .lookup(&changed_difficulty, Some("work"), &network) .is_none()); + let mut changed_merkle_root = original.clone(); + Arc::make_mut(&mut changed_merkle_root.header).merkle_root.0[0] ^= 1; + assert!(cache + .lookup(&changed_merkle_root, Some("work"), &network) + .is_none()); + let mut changed_transactions = original; changed_transactions .transactions @@ -283,7 +298,7 @@ mod tests { } #[test] - fn inserting_a_reused_work_id_replaces_the_old_candidate() { + fn inserting_a_reused_work_id_does_not_replace_the_old_candidate() { let network = Network::Mainnet; let original = test_block(); let mut replacement = original.clone(); @@ -303,8 +318,8 @@ mod tests { &network, ); - assert!(cache.lookup(&replacement, Some("work"), &network).is_some()); - assert!(cache.lookup(&original, Some("work"), &network).is_none()); + assert!(cache.lookup(&replacement, Some("work"), &network).is_none()); + assert!(cache.lookup(&original, Some("work"), &network).is_some()); assert_eq!( cache .0 diff --git a/crates/zakura-consensus/src/block/tests.rs b/crates/zakura-consensus/src/block/tests.rs index e5c6737d58..5c0b963d43 100644 --- a/crates/zakura-consensus/src/block/tests.rs +++ b/crates/zakura-consensus/src/block/tests.rs @@ -2,9 +2,10 @@ #![allow(clippy::unwrap_in_result)] +use chrono::DateTime; use color_eyre::eyre::{eyre, Report}; use once_cell::sync::Lazy; -use tower::{buffer::Buffer, util::BoxService, ServiceExt}; +use tower::{buffer::Buffer, service_fn, util::BoxService, Service, ServiceExt}; use zakura_chain::{amount::NegativeAllowed, ironwood}; use zakura_chain::{ @@ -134,6 +135,238 @@ static INVALID_COINBASE_TRANSCRIPT: Lazy< ] }); +fn prepared_test_verifier( + network: &Network, +) -> impl Service { + let state = service_fn(|request: zs::Request| async move { + let response = match request { + zs::Request::KnownBlock(_) => zs::Response::KnownBlock(None), + zs::Request::CheckBlockProposalValidity(_) => zs::Response::ValidBlockProposal, + _ => panic!("prepared-path test received an unexpected state request: {request:?}"), + }; + Ok::<_, BoxError>(response) + }); + let transaction = + service_fn(|request| async move { Ok::<_, BoxError>(accept_block_transaction(request)) }); + + SemanticBlockVerifier::new(network, state, transaction) +} + +fn accept_block_transaction(request: tx::Request) -> tx::Response { + let tx::Request::Block { transaction, .. } = request else { + panic!("prepared-path test received a mempool transaction request"); + }; + let miner_fee = (!transaction.is_coinbase()).then(Amount::zero); + tx::Response::Block { + tx_id: transaction.as_ref().into(), + miner_fee, + sigops: 0, + } +} + +async fn prepare_for_test(verifier: &mut V, block: Arc) +where + V: Service, +{ + verifier + .ready() + .await + .expect("the verifier is ready") + .call(Request::Prepare { + block, + work_id: Some("work".to_owned()), + }) + .await + .expect("the candidate prepares successfully"); +} + +fn nu5_prepared_test_block(network: &Network, lock_time: Option) -> Block { + let height = Height(1); + let mut block = + Block::zcash_deserialize(&zakura_test::vectors::BLOCK_MAINNET_GENESIS_BYTES[..]) + .expect("the genesis block deserializes"); + block.transactions = vec![Arc::new(v5_coinbase_transaction( + NetworkUpgrade::Nu5, + height, + network, + ))]; + if let Some(lock_time) = lock_time { + block.transactions.push(Arc::new(Transaction::V5 { + network_upgrade: NetworkUpgrade::Nu5, + lock_time, + expiry_height: height, + inputs: vec![transparent::Input::PrevOut { + outpoint: transparent::OutPoint { + hash: zakura_chain::transaction::Hash([1; 32]), + index: 0, + }, + unlock_script: transparent::Script::new(&[]), + sequence: 0, + }], + outputs: vec![], + sapling_shielded_data: None, + orchard_shielded_data: None, + })); + } + Arc::make_mut(&mut block.header).merkle_root = block.transactions.iter().collect(); + block +} + +#[tokio::test] +async fn prepared_mined_commit_rechecks_equihash() { + let _init_guard = zakura_test::init(); + let network = Network::Mainnet; + let candidate = Arc::new( + Block::zcash_deserialize(&zakura_test::vectors::BLOCK_MAINNET_GENESIS_BYTES[..]) + .expect("the genesis block deserializes"), + ); + let mut verifier = prepared_test_verifier(&network); + prepare_for_test(&mut verifier, candidate.clone()).await; + + let mut solved = (*candidate).clone(); + Arc::make_mut(&mut solved.header).solution = + zakura_chain::work::equihash::Solution::for_proposal_for_network(&network); + let height = solved + .coinbase_height() + .expect("the candidate has a coinbase height"); + for nonce in 0u32.. { + Arc::make_mut(&mut solved.header).nonce.0[..4].copy_from_slice(&nonce.to_le_bytes()); + let hash = solved.hash(); + if check::difficulty_is_valid(&solved.header, &network, &height, &hash).is_ok() { + break; + } + } + let result = verifier + .ready() + .await + .expect("the verifier is ready") + .call(Request::CommitMined { + block: Arc::new(solved), + work_id: Some("work".to_owned()), + admission: zs::BlockAdmission::pending(), + }) + .await; + + assert!(matches!(result, Err(VerifyBlockError::Equihash { .. }))); +} + +#[tokio::test] +async fn prepared_mined_commit_rechecks_header_time() { + let _init_guard = zakura_test::init(); + let network = librustzcash_conversion_test_network(NetworkUpgrade::Nu5); + let candidate = Arc::new(nu5_prepared_test_block(&network, None)); + let mut verifier = prepared_test_verifier(&network); + prepare_for_test(&mut verifier, candidate.clone()).await; + + let mut solved = (*candidate).clone(); + Arc::make_mut(&mut solved.header).time = Utc::now() + .checked_add_signed(chrono::Duration::hours(3)) + .expect("three hours fits in the supported time range"); + let result = verifier + .ready() + .await + .expect("the verifier is ready") + .call(Request::CommitMined { + block: Arc::new(solved), + work_id: Some("work".to_owned()), + admission: zs::BlockAdmission::pending(), + }) + .await; + + assert!(matches!(result, Err(VerifyBlockError::Time(_)))); +} + +#[tokio::test] +async fn prepared_mined_commit_rechecks_transaction_lock_time() { + let _init_guard = zakura_test::init(); + let network = librustzcash_conversion_test_network(NetworkUpgrade::Nu5); + let unlock_time = DateTime::from_timestamp(Utc::now().timestamp() - 60, 0) + .expect("the recent timestamp is valid"); + let mut candidate = nu5_prepared_test_block(&network, Some(LockTime::Time(unlock_time))); + Arc::make_mut(&mut candidate.header).time = unlock_time + chrono::Duration::seconds(1); + let candidate = Arc::new(candidate); + let mut verifier = prepared_test_verifier(&network); + prepare_for_test(&mut verifier, candidate.clone()).await; + + let mut solved = (*candidate).clone(); + Arc::make_mut(&mut solved.header).time = unlock_time; + let result = verifier + .ready() + .await + .expect("the verifier is ready") + .call(Request::CommitMined { + block: Arc::new(solved), + work_id: Some("work".to_owned()), + admission: zs::BlockAdmission::pending(), + }) + .await; + + assert!(matches!( + result, + Err(VerifyBlockError::Transaction( + TransactionError::LockedUntilAfterBlockTime(_) + )) + )); +} + +#[tokio::test] +async fn failed_preparation_does_not_populate_the_cache() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + let _init_guard = zakura_test::init(); + let network = librustzcash_conversion_test_network(NetworkUpgrade::Nu5); + let candidate = Arc::new(nu5_prepared_test_block(&network, None)); + let transaction_calls = Arc::new(AtomicUsize::new(0)); + let transaction = service_fn({ + let transaction_calls = transaction_calls.clone(); + move |request| { + transaction_calls.fetch_add(1, Ordering::Relaxed); + async move { Ok::<_, BoxError>(accept_block_transaction(request)) } + } + }); + let state = service_fn(|request: zs::Request| async move { + match request { + zs::Request::KnownBlock(_) => Ok(zs::Response::KnownBlock(None)), + zs::Request::CheckBlockProposalValidity(_) => { + Err(std::io::Error::other("proposal rejected").into()) + } + zs::Request::CommitSemanticallyVerifiedBlockWithAdmission { block, .. } => { + Ok(zs::Response::Committed(block.hash)) + } + _ => panic!("failed-preparation test received an unexpected request: {request:?}"), + } + }); + let mut verifier = SemanticBlockVerifier::new(&network, state, transaction); + + let prepare_result = verifier + .ready() + .await + .expect("the verifier is ready") + .call(Request::Prepare { + block: candidate.clone(), + work_id: Some("work".to_owned()), + }) + .await; + assert!(matches!( + prepare_result, + Err(VerifyBlockError::ValidateProposal(_)) + )); + assert_eq!(transaction_calls.load(Ordering::Relaxed), 1); + + let commit_result = verifier + .ready() + .await + .expect("the verifier is ready") + .call(Request::CommitMined { + block: candidate, + work_id: Some("work".to_owned()), + admission: zs::BlockAdmission::pending(), + }) + .await; + assert!(commit_result.is_ok()); + assert_eq!(transaction_calls.load(Ordering::Relaxed), 2); +} + // TODO: enable this test after implementing contextual verification // #[tokio::test] // #[ignore] diff --git a/crates/zakura-rpc/src/methods.rs b/crates/zakura-rpc/src/methods.rs index cb6abe3610..7f3d0727cf 100644 --- a/crates/zakura-rpc/src/methods.rs +++ b/crates/zakura-rpc/src/methods.rs @@ -141,6 +141,9 @@ use types::{ z_validate_address::ZValidateAddressResponse, }; +/// Bounds the final mined-block event send after the RPC lifecycle has detached. +const MINED_BLOCK_EVENT_SEND_TIMEOUT: Duration = Duration::from_secs(5); + /// Calls a Tower service and maps readiness or call errors to /// [`server::error::LegacyCode::Misc`]. async fn call_service(service: S, request: Request) -> Result @@ -1044,35 +1047,29 @@ where } fn prepare_template_in_background(&self, template: &BlockTemplateResponse) { - #[cfg(test)] - { - let _ = (template, self.gbt.try_acquire_template_preparation()); - } - - #[cfg(not(test))] - { - let Some(preparation_permit) = self.gbt.try_acquire_template_preparation() else { - metrics::counter!("mining.template_preparation.saturated").increment(1); - return; - }; - let Ok(block) = proposal_block_from_template(template, None, &self.network) else { - return; - }; - let request = zakura_consensus::Request::Prepare { - block: Arc::new(block), - work_id: Some(template.work_id().clone()), - }; - let verifier = self.gbt.block_verifier_router(); - tokio::spawn( - async move { - let _preparation_permit = preparation_permit; - if let Err(error) = verifier.oneshot(request).await { - tracing::debug!(?error, "background mining candidate preparation failed"); - } + let Some(preparation_permit) = self.gbt.try_acquire_template_preparation() else { + metrics::counter!("mining.template_preparation.saturated").increment(1); + return; + }; + let template = template.clone(); + let network = self.network.clone(); + let verifier = self.gbt.block_verifier_router(); + tokio::spawn( + async move { + let _preparation_permit = preparation_permit; + let Ok(block) = proposal_block_from_template(&template, None, &network) else { + return; + }; + let request = zakura_consensus::Request::Prepare { + block: Arc::new(block), + work_id: Some(template.work_id().clone()), + }; + if let Err(error) = verifier.oneshot(request).await { + tracing::debug!(?error, "background mining candidate preparation failed"); } - .in_current_span(), - ); - } + } + .in_current_span(), + ); } /// Sets the end-of-support height reported by `getdeprecationinfo`. @@ -2827,6 +2824,7 @@ where let admission_start = std::time::Instant::now(); let mut early_result = None; + let mut pending_registration = None; let verification_result = tokio::select! { biased; @@ -2836,19 +2834,19 @@ where if admitted && admission.optimistic_relay_authorized() && optimistic_block_inventory - && pending_blocks.insert(block.clone()) { - let (advertised, receiver) = tokio::sync::oneshot::channel(); - let event = MinedBlockEvent::Early { - hash: block_hash, - height, - submitted_at, - advertised, - }; - if mined_block_sender.try_send(event).is_ok() { - early_result = Some(receiver); - } else { - pending_blocks.resolve(block_hash, Err(())); + if let Some(registration) = pending_blocks.insert(block.clone()) { + let (advertised, receiver) = tokio::sync::oneshot::channel(); + let event = MinedBlockEvent::Early { + hash: block_hash, + height, + submitted_at, + advertised, + }; + if mined_block_sender.try_send(event).is_ok() { + early_result = Some(receiver); + pending_registration = Some(registration); + } } } verification.await @@ -2856,13 +2854,14 @@ where result = &mut verification => result, }; - pending_blocks.resolve( - block_hash, - verification_result - .as_ref() - .map(|_| block.clone()) - .map_err(|_| ()), - ); + if let Some(registration) = pending_registration { + registration.resolve( + verification_result + .as_ref() + .map(|_| block.clone()) + .map_err(|_| ()), + ); + } let committed = verification_result.is_ok(); tokio::spawn(async move { @@ -2899,7 +2898,20 @@ where early_advertised, } }; - let _ = mined_block_sender.send(event).await; + let send_result = tokio::time::timeout( + MINED_BLOCK_EVENT_SEND_TIMEOUT, + mined_block_sender.send(event), + ) + .await; + if !matches!(send_result, Ok(Ok(()))) { + metrics::counter!("mining.optimistic_inventory.final_send_failures") + .increment(1); + tracing::warn!( + ?block_hash, + ?height, + "could not send the final mined-block event" + ); + } }); verification_result }); diff --git a/crates/zakura-rpc/src/methods/types/submit_block.rs b/crates/zakura-rpc/src/methods/types/submit_block.rs index 672abe7a8d..7d1635c890 100644 --- a/crates/zakura-rpc/src/methods/types/submit_block.rs +++ b/crates/zakura-rpc/src/methods/types/submit_block.rs @@ -2,7 +2,10 @@ use std::{ collections::HashMap, - sync::{Arc, Mutex}, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, Mutex, + }, time::Duration, }; @@ -84,6 +87,7 @@ enum PendingStatus { #[derive(Debug)] struct PendingBlock { + owner_id: u64, status: watch::Sender, } @@ -92,6 +96,7 @@ struct PendingBlock { struct PendingBlockRegistryInner { entries: Mutex>, wait_permits: Arc, + next_owner_id: AtomicU64, } /// Holds early-advertised block bodies until their contextual commits finish. @@ -103,15 +108,41 @@ impl Default for PendingBlockRegistry { Self(Arc::new(PendingBlockRegistryInner { entries: Mutex::new(HashMap::new()), wait_permits: Arc::new(Semaphore::new(MAX_PENDING_BLOCK_WAITS)), + next_owner_id: AtomicU64::new(1), })) } } +/// Owns one pending-block registry entry. +#[derive(Debug)] +pub(crate) struct PendingBlockRegistration { + registry: PendingBlockRegistry, + hash: block::Hash, + owner_id: u64, + resolved: bool, +} + +impl PendingBlockRegistration { + /// Resolves this registration and wakes its peer waiters. + pub(crate) fn resolve(mut self, result: Result, ()>) { + self.registry.resolve(self.hash, self.owner_id, result); + self.resolved = true; + } +} + +impl Drop for PendingBlockRegistration { + fn drop(&mut self) { + if !self.resolved { + self.registry.resolve(self.hash, self.owner_id, Err(())); + } + } +} + impl PendingBlockRegistry { /// Inserts a block before its early inventory is sent. /// - /// Returns false when the bounded registry is full. - pub fn insert(&self, block: Arc) -> bool { + /// Returns no registration when the hash already has an owner or the registry is full. + pub(crate) fn insert(&self, block: Arc) -> Option { let hash = block.hash(); let mut entries = self .0 @@ -119,29 +150,41 @@ impl PendingBlockRegistry { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); if entries.contains_key(&hash) { - return false; + return None; } if entries.len() >= MAX_PENDING_BLOCKS { metrics::counter!("mining.pending_registry.saturated").increment(1); - return false; + return None; } + let owner_id = self.0.next_owner_id.fetch_add(1, Ordering::Relaxed); let (status, _receiver) = watch::channel(PendingStatus::Waiting); - entries.insert(hash, PendingBlock { status }); - true + entries.insert(hash, PendingBlock { owner_id, status }); + Some(PendingBlockRegistration { + registry: self.clone(), + hash, + owner_id, + resolved: false, + }) } - /// Resolves peer waiters and removes a terminal entry. - pub fn resolve(&self, hash: block::Hash, result: Result, ()>) { - let entry = self + fn resolve(&self, hash: block::Hash, owner_id: u64, result: Result, ()>) { + let mut entries = self .0 .entries .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .remove(&hash); - let Some(entry) = entry else { + .unwrap_or_else(std::sync::PoisonError::into_inner); + if !entries + .get(&hash) + .is_some_and(|entry| entry.owner_id == owner_id) + { return; - }; + } + let entry = entries + .remove(&hash) + .expect("entry exists because its owner matched under the same lock"); + drop(entries); + let status = match result { Ok(block) => PendingStatus::Committed(block), Err(()) => PendingStatus::Failed, @@ -161,21 +204,17 @@ impl PendingBlockRegistry { .unwrap_or_else(std::sync::PoisonError::into_inner) .get(&hash) .map(|entry| entry.status.subscribe()); - let wait_permit = status.as_ref().and_then(|_| { - self.0 - .wait_permits - .clone() - .try_acquire_owned() - .map_err(|_| { - metrics::counter!("mining.pending_peer_wait.saturated").increment(1); - }) - .ok() - }); + let wait_permits = status.as_ref().map(|_| self.0.wait_permits.clone()); let deadline = tokio::time::Instant::now() + PENDING_BLOCK_WAIT; async move { let mut status = status?; - let _wait_permit: OwnedSemaphorePermit = wait_permit?; + let _wait_permit: OwnedSemaphorePermit = wait_permits? + .try_acquire_owned() + .map_err(|_| { + metrics::counter!("mining.pending_peer_wait.saturated").increment(1); + }) + .ok()?; let start = std::time::Instant::now(); let result = tokio::time::timeout_at(deadline, async { loop { @@ -297,14 +336,17 @@ mod tests { let registry = PendingBlockRegistry::default(); let block = test_block(); let hash = block.hash(); - assert!(registry.insert(block.clone())); + let registration = registry + .insert(block.clone()) + .expect("the registry accepts the block"); + assert!(registry.insert(block.clone()).is_none()); let wait = tokio::spawn({ let registry = registry.clone(); async move { registry.wait(hash).await } }); tokio::task::yield_now().await; - registry.resolve(hash, Ok(block.clone())); + registration.resolve(Ok(block.clone())); assert_eq!(wait.await.expect("wait task succeeds"), Some(block)); } @@ -314,10 +356,12 @@ mod tests { let registry = PendingBlockRegistry::default(); let block = test_block(); let hash = block.hash(); - assert!(registry.insert(block.clone())); + let registration = registry + .insert(block.clone()) + .expect("the registry accepts the block"); let wait = registry.wait(hash); - registry.resolve(hash, Ok(block.clone())); + registration.resolve(Ok(block.clone())); assert_eq!(wait.await, Some(block)); } @@ -327,14 +371,16 @@ mod tests { let registry = PendingBlockRegistry::default(); let block = test_block(); let hash = block.hash(); - assert!(registry.insert(block)); + let registration = registry + .insert(block) + .expect("the registry accepts the block"); let wait = tokio::spawn({ let registry = registry.clone(); async move { registry.wait(hash).await } }); tokio::task::yield_now().await; - registry.resolve(hash, Err(())); + registration.resolve(Err(())); assert_eq!(wait.await.expect("wait task succeeds"), None); } @@ -344,16 +390,27 @@ mod tests { let registry = PendingBlockRegistry::default(); let block = test_block(); let hash = block.hash(); - assert!(registry.insert(block.clone())); + let registration = registry + .insert(block.clone()) + .expect("the registry accepts the block"); let waits: Vec<_> = (0..MAX_PENDING_BLOCK_WAITS) - .map(|_| registry.wait(hash)) + .map(|_| { + let registry = registry.clone(); + tokio::spawn(async move { registry.wait(hash).await }) + }) .collect(); + while registry.0.wait_permits.available_permits() > 0 { + tokio::task::yield_now().await; + } assert_eq!(registry.wait(hash).await, None); - drop(waits); + for wait in waits { + wait.abort(); + let _ = wait.await; + } let wait = registry.wait(hash); - registry.resolve(hash, Ok(block.clone())); + registration.resolve(Ok(block.clone())); assert_eq!(wait.await, Some(block)); } @@ -361,15 +418,21 @@ mod tests { fn pending_registry_is_bounded() { let registry = PendingBlockRegistry::default(); let original = test_block(); + let mut registrations = Vec::new(); for nonce in 0..MAX_PENDING_BLOCKS { let mut block = (*original).clone(); let nonce = u8::try_from(nonce).expect("the registry bound fits in u8"); Arc::make_mut(&mut block.header).nonce = [nonce; 32].into(); - assert!(registry.insert(Arc::new(block))); + registrations.push( + registry + .insert(Arc::new(block)) + .expect("the registry has capacity"), + ); } let mut overflow = (*original).clone(); Arc::make_mut(&mut overflow.header).nonce = [u8::MAX; 32].into(); - assert!(!registry.insert(Arc::new(overflow))); + assert!(registry.insert(Arc::new(overflow)).is_none()); + drop(registrations); } } diff --git a/crates/zakura-state/src/service.rs b/crates/zakura-state/src/service.rs index 8d854d2ab2..b2b5f4f95f 100644 --- a/crates/zakura-state/src/service.rs +++ b/crates/zakura-state/src/service.rs @@ -919,7 +919,7 @@ impl StateService { fn queue_and_commit_to_non_finalized_state( &mut self, semantically_verified: SemanticallyVerifiedBlock, - admission: Option, + mut admission: Option, ) -> oneshot::Receiver> { tracing::debug!(block = %semantically_verified.block, "queueing block for contextual verification"); let parent_hash = semantically_verified.block.header.previous_block_hash; @@ -966,16 +966,17 @@ impl StateService { // [`Request::CommitSemanticallyVerifiedBlock`] contract: a request to commit a block which // has been queued but not yet committed to the state fails the older request and replaces // it with the newer request. - let rsp_rx = if let Some((_, old_rsp_tx, _)) = self + let rsp_rx = if let Some((_, old_rsp_tx, old_admission)) = self .non_finalized_state_queued_blocks .get_mut(&semantically_verified.hash) { - if let Some(admission) = admission { - admission.reject(); - } tracing::debug!("replacing older queued request with new request"); let (mut rsp_tx, rsp_rx) = oneshot::channel(); std::mem::swap(old_rsp_tx, &mut rsp_tx); + std::mem::swap(old_admission, &mut admission); + if let Some(admission) = admission { + admission.reject(); + } let _ = rsp_tx.send(Err(CommitBlockError::new_duplicate( Some(semantically_verified.hash.into()), KnownBlock::Queue, diff --git a/crates/zakura-state/src/service/tests.rs b/crates/zakura-state/src/service/tests.rs index 796668c4d7..3e39081a4c 100644 --- a/crates/zakura-state/src/service/tests.rs +++ b/crates/zakura-state/src/service/tests.rs @@ -31,13 +31,52 @@ use crate::{ StateService, }, tests::setup::{partial_nu5_chain_strategy, transaction_v4_from_coinbase}, - BoxError, CheckpointVerifiedBlock, Config, HistoricalTreeUnavailable, PruningConfig, Request, - Response, SemanticallyVerifiedBlock, StateInitError, StorageMode, ValidateContextError, - CHAIN_TIP_UPDATE_WAIT_LIMIT, MAX_HISTORICAL_TREE_REPLAY_BLOCKS, + BlockAdmission, BoxError, CheckpointVerifiedBlock, Config, HistoricalTreeUnavailable, + PruningConfig, Request, Response, SemanticallyVerifiedBlock, StateInitError, StorageMode, + ValidateContextError, CHAIN_TIP_UPDATE_WAIT_LIMIT, MAX_HISTORICAL_TREE_REPLAY_BLOCKS, }; const LAST_BLOCK_HEIGHT: u32 = 10; +#[test] +fn queued_duplicate_replaces_the_old_admission() { + let _init_guard = zakura_test::init(); + let network = Network::Mainnet; + let runtime = Runtime::new().expect("the Tokio runtime starts"); + let (mut state_service, _, _, _) = runtime + .block_on(StateService::new( + Config::ephemeral(), + &network, + Height::MAX, + 0, + )) + .expect("ephemeral state initialization succeeds"); + let block = Arc::new( + zakura_test::vectors::BLOCK_MAINNET_1_BYTES + .zcash_deserialize_into::() + .expect("the mainnet height-one block is valid"), + ) + .prepare(); + + let old_admission = BlockAdmission::pending(); + let old_response = state_service + .queue_and_commit_to_non_finalized_state(block.clone(), Some(old_admission.clone())); + let new_admission = BlockAdmission::pending(); + let _new_response = state_service + .queue_and_commit_to_non_finalized_state(block.clone(), Some(new_admission.clone())); + + assert!(!runtime.block_on(old_admission.wait())); + assert!(old_response + .blocking_recv() + .expect("the replaced request receives a response") + .is_err()); + let queued = state_service + .non_finalized_state_queued_blocks + .get_mut(&block.hash) + .expect("the newer request remains queued"); + assert_eq!(queued.2.as_ref(), Some(&new_admission)); +} + fn prepared_relay_test_state() -> ( Network, super::finalized_state::FinalizedState, diff --git a/crates/zakurad/src/components/inbound.rs b/crates/zakurad/src/components/inbound.rs index 709ef841f9..1e85dd73e0 100644 --- a/crates/zakurad/src/components/inbound.rs +++ b/crates/zakurad/src/components/inbound.rs @@ -52,12 +52,15 @@ mod tests; use downloads::{Downloads as BlockDownloads, GossipedTipChildHeightMismatch}; -/// The maximum amount of time an inbound service response can take. +/// The maximum response time for block-body requests that can wait for a mined-block commit. /// /// If the response takes longer than this time, it will be cancelled, /// and the peer might be disconnected. pub const MAX_INBOUND_RESPONSE_TIME: Duration = Duration::from_secs(18); +/// The maximum response time for requests that do not wait for a mined-block commit. +const DEFAULT_INBOUND_RESPONSE_TIME: Duration = Duration::from_secs(5); + /// The number of bytes the [`Inbound`] service will queue in response to a single block or /// transaction request, before ignoring any additional block or transaction IDs in that request. /// @@ -197,15 +200,11 @@ async fn retained_block_height(mut state: State, hash: block::Hash) -> Option Result>, zn::BoxError> { - // Subscribe before the state lookup. A commit can remove the registry entry while state - // answers this request. - let pending_wait = pending_blocks.wait(hash); let response = state .ready() .await? @@ -214,7 +213,7 @@ async fn block_by_hash_or_pending( match response { zs::Response::Block(Some(block)) => Ok(Some(block)), - zs::Response::Block(None) => Ok(pending_wait.await), + zs::Response::Block(None) => Ok(None), _ => unreachable!("wrong response from state"), } } @@ -597,7 +596,16 @@ impl Service for Inbound { } }; - match req { + let response_timeout = if matches!( + &req, + zn::Request::BlocksByHash(_) | zn::Request::BlocksByHashFrom { .. } + ) { + MAX_INBOUND_RESPONSE_TIME + } else { + DEFAULT_INBOUND_RESPONSE_TIME + }; + + let response = match req { zn::Request::Peers => { // # Security // @@ -639,28 +647,38 @@ impl Service for Inbound { async move { let mut blocks: Vec, Option), block::Hash>> = Vec::new(); let mut total_size = 0; - let mut lookups = FuturesUnordered::new(); + let mut state_lookup_bytes = 0; + let mut pending_lookups = FuturesUnordered::new(); + let mut lookup_results = Vec::new(); for (index, &hash) in hashes.iter().take(GETDATA_MAX_BLOCK_COUNT).enumerate() { - let state = state.clone(); - let pending_blocks = pending_blocks.clone(); - lookups.push(async move { - let block = - block_by_hash_or_pending(state, pending_blocks, hash).await?; - Ok::<_, zn::BoxError>((index, hash, block)) - }); + if state_lookup_bytes >= GETDATA_SENT_BYTES_LIMIT { + break; + } + + // Subscribe before the state lookup. A commit can remove the registry entry + // while state answers this request. + let pending_wait = pending_blocks.wait(hash); + match block_by_hash(state.clone(), hash).await? { + Some(block) => { + state_lookup_bytes = state_lookup_bytes + .saturating_add(block.zcash_serialized_size()); + lookup_results.push((index, hash, Some(block))); + } + None => pending_lookups.push(async move { + (index, hash, pending_wait.await) + }), + } } - // Start every state lookup and pending wait before awaiting any result. - let mut lookup_results = Vec::with_capacity(lookups.len()); - while let Some(result) = lookups.next().await { - lookup_results.push(result?); + while let Some(result) = pending_lookups.next().await { + lookup_results.push(result); } lookup_results.sort_unstable_by_key(|(index, _, _)| *index); for (_, hash, block) in lookup_results { if total_size >= GETDATA_SENT_BYTES_LIMIT { - continue; + break; } // Add the block responses to the list, while updating the size limit. @@ -837,6 +855,14 @@ impl Service for Inbound { } zn::Request::AdvertiseBlockToAll(_) => unreachable!("should always be decoded as `AdvertiseBlock` request") + }; + + async move { + match tokio::time::timeout(response_timeout, response).await { + Ok(response) => response, + Err(error) => Err(Box::new(error) as zn::BoxError), + } } + .boxed() } } diff --git a/crates/zakurad/src/components/inbound/tests.rs b/crates/zakurad/src/components/inbound/tests.rs index d1610425cb..e5e1563dd5 100644 --- a/crates/zakurad/src/components/inbound/tests.rs +++ b/crates/zakurad/src/components/inbound/tests.rs @@ -6,7 +6,7 @@ use std::{ }; use super::{ - block_by_hash_or_pending, block_misbehavior, canonical_ip, PrunedBlockNotFoundLogger, + block_by_hash, block_misbehavior, canonical_ip, PrunedBlockNotFoundLogger, ZCASHD_COMPAT_PRUNED_BLOCK_LOG_INTERVAL, }; @@ -16,8 +16,6 @@ async fn peer_block_lookup_queries_all_active_chains() { use tower::{buffer::Buffer, util::BoxService}; use zakura_chain::{block::Block, serialization::ZcashDeserializeInto}; - use zakura_rpc::PendingBlockRegistry; - let block: Arc = zakura_test::vectors::BLOCK_MAINNET_GENESIS_BYTES .zcash_deserialize_into() .expect("the genesis block is valid"); @@ -33,7 +31,7 @@ async fn peer_block_lookup_queries_all_active_chains() { let state = Buffer::new(BoxService::new(state), 1); assert_eq!( - block_by_hash_or_pending(state, PendingBlockRegistry::default(), hash) + block_by_hash(state, hash) .await .expect("the state lookup succeeds"), Some(block), diff --git a/docs/changelog/params.md b/docs/changelog/params.md index 7f5d353d40..8d0128dd49 100644 --- a/docs/changelog/params.md +++ b/docs/changelog/params.md @@ -28,6 +28,16 @@ Keep entries **newest-first**. Each row records: | Parameter | Location | Old → New | PR | Why | | --- | --- | --- | --- | --- | +| `MINED_BLOCK_EVENT_SEND_TIMEOUT` | `crates/zakura-rpc/src/methods.rs` | new → `5 s` | [#748](https://github.com/zakura-core/zakura/pull/748) | Bound detached final gossip sends when the mined-block event channel stalls. | +| `MAX_BACKGROUND_TEMPLATE_PREPARATIONS` | `crates/zakura-rpc/src/methods/types/get_block_template.rs` | new → `1` task | [#748](https://github.com/zakura-core/zakura/pull/748) | Prevent repeated template requests from delaying solved block submissions with concurrent preparation work. | +| `ENTRY_TTL` | `crates/zakura-consensus/src/block/prepared.rs` | new → `10 min` | [#748](https://github.com/zakura-core/zakura/pull/748) | Expire prepared candidates after miners should have replaced their work. | +| `MAX_BYTES` | `crates/zakura-consensus/src/block/prepared.rs` | new → `64 MiB` | [#748](https://github.com/zakura-core/zakura/pull/748) | Bound prepared-candidate memory separately from the entry count. | +| `MAX_ENTRIES` | `crates/zakura-consensus/src/block/prepared.rs` | new → `32` candidates | [#748](https://github.com/zakura-core/zakura/pull/748) | Retain recent mining candidates while bounding verification-cache work and memory. | +| `MAX_PENDING_BLOCK_WAITS` | `crates/zakura-rpc/src/methods/types/submit_block.rs` | new → `32` waits | [#748](https://github.com/zakura-core/zakura/pull/748) | Reserve inbound capacity when peers request early-advertised blocks. | +| `MAX_PENDING_BLOCKS` | `crates/zakura-rpc/src/methods/types/submit_block.rs` | new → `16` blocks | [#748](https://github.com/zakura-core/zakura/pull/748) | Bound mined blocks waiting for contextual commit. | +| `PENDING_BLOCK_WAIT` | `crates/zakura-rpc/src/methods/types/submit_block.rs` | new → `15 s` | [#748](https://github.com/zakura-core/zakura/pull/748) | Let peers wait for early-advertised blocks below the legacy peer request timeout. | +| `MAX_INBOUND_RESPONSE_TIME` | `crates/zakurad/src/components/inbound.rs` | `5 s` → `18 s` for block-body requests | [#748](https://github.com/zakura-core/zakura/pull/748) | Give pending block-body requests time to settle while other inbound requests keep the 5-second timeout. | +| `mining.optimistic_block_inventory` | `crates/zakura-rpc/src/config/mining.rs` | new → `true` | [#748](https://github.com/zakura-core/zakura/pull/748) | Enable inventory after prepared-work validation and state admission while allowing operators to restore commit-first relay. | | `MAX_HISTORICAL_TREE_REPLAY_BLOCKS` | `crates/zakura-state/src/constants.rs` | `DEFAULT_MAX_HISTORICAL_TREE_REPLAY_BLOCKS = 4_000_000` → `100_000` | [#775](https://github.com/zakura-core/zakura/pull/775) | Use one bound for startup grid-gap validation and per-request replay, preventing a grid whose anchors fail verification from falling back to replaying the entire absent band. | | `TLS_HANDSHAKE_TIMEOUT` | `crates/zakura-rpc/src/indexer/server.rs` | new → `10 s` | [#596](https://github.com/zakura-core/zakura/pull/596) | Drop indexer connections that do not complete the unauthenticated TLS handshake promptly, so stalled handshakes cannot retain a bounded connection slot indefinitely. | | `MAX_CONCURRENT_STREAMS_PER_CONNECTION` | `crates/zakura-rpc/src/indexer/server.rs` | new → `64` streams | [#596](https://github.com/zakura-core/zakura/pull/596) | Let trusted indexers multiplex long-lived subscriptions, parallel block-range backfills, and unary queries on one HTTP/2 connection while retaining a finite per-connection task and response-buffer bound. | diff --git a/docs/changelog/unreleased/748.md b/docs/changelog/unreleased/748.md index 46fb39c5bc..ee81c1958f 100644 --- a/docs/changelog/unreleased/748.md +++ b/docs/changelog/unreleased/748.md @@ -1,5 +1,6 @@ ## Added -- Added early inventory for prepared mined blocks after expected-work validation and state - admission. -- Added prepared mining-candidate reuse through `workid`. +- Zakura now advertises prepared mined blocks after expected-work validation and state admission + ([#748](https://github.com/zakura-core/zakura/pull/748)). +- Zakura now reuses prepared mining candidates through `workid` + ([#748](https://github.com/zakura-core/zakura/pull/748)). From 522c70118e089e411ae1baa6ad2df3022f1b4e5e Mon Sep 17 00:00:00 2001 From: evan-forbes Date: Mon, 24 Aug 2026 14:37:49 -0500 Subject: [PATCH 16/22] docs(mining): clarify hardening invariants --- crates/zakura-consensus/src/block/prepared.rs | 14 ++++++++++++++ crates/zakura-state/src/request.rs | 7 +++++++ crates/zakurad/src/components/inbound.rs | 3 +++ 3 files changed, 24 insertions(+) diff --git a/crates/zakura-consensus/src/block/prepared.rs b/crates/zakura-consensus/src/block/prepared.rs index 74d0d19f30..35cfce0f6c 100644 --- a/crates/zakura-consensus/src/block/prepared.rs +++ b/crates/zakura-consensus/src/block/prepared.rs @@ -56,6 +56,20 @@ impl PreparedCandidateCache { work_id: Option<&str>, network: &Network, ) -> Option { + // Deriving the candidate bytes costs a full block serialization, so skip it when the + // cache holds no entry that could match. + { + let mut inner = self + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + inner.prune_expired(); + if inner.entries.is_empty() { + metrics::counter!("mining.prepared_cache.misses").increment(1); + return None; + } + } + let immutable_bytes = immutable_candidate_bytes(block, network); let fingerprint = fingerprint(&immutable_bytes); let mut inner = self diff --git a/crates/zakura-state/src/request.rs b/crates/zakura-state/src/request.rs index 6f9c6ba7bd..ad98287d95 100644 --- a/crates/zakura-state/src/request.rs +++ b/crates/zakura-state/src/request.rs @@ -114,6 +114,13 @@ impl BlockAdmission { } /// Waits until state admits or rejects the block. + /// + /// # Correctness + /// + /// This future never resolves when neither `admit` nor `reject` runs. The state rejects + /// duplicates, queue replacements, and expired blocks, but a verifier error before the state + /// receives the block leaves the admission pending. Callers must await this future under a + /// cancellation path, such as a `select!` arm that also awaits verification. pub async fn wait(&self) -> bool { loop { let notified = self.0.changed.notified(); diff --git a/crates/zakurad/src/components/inbound.rs b/crates/zakurad/src/components/inbound.rs index 1e85dd73e0..33c90d6553 100644 --- a/crates/zakurad/src/components/inbound.rs +++ b/crates/zakurad/src/components/inbound.rs @@ -56,6 +56,9 @@ use downloads::{Downloads as BlockDownloads, GossipedTipChildHeightMismatch}; /// /// If the response takes longer than this time, it will be cancelled, /// and the peer might be disconnected. +/// +/// This constant must exceed the 15-second `PENDING_BLOCK_WAIT` in `zakura-rpc`, so that a peer +/// waiting for an early-advertised block reaches that wait's own timeout first. pub const MAX_INBOUND_RESPONSE_TIME: Duration = Duration::from_secs(18); /// The maximum response time for requests that do not wait for a mined-block commit. From 94704857b88af9e9f6f2efb6b69f40b926002adf Mon Sep 17 00:00:00 2001 From: evan-forbes Date: Mon, 24 Aug 2026 17:21:17 -0500 Subject: [PATCH 17/22] fix(state): exclude proposals from contextual metrics --- .../src/service/non_finalized_state.rs | 98 +++++++++++-------- .../src/service/non_finalized_state/tests.rs | 79 +++++++++++++++ crates/zakura-state/src/service/write.rs | 48 ++++----- 3 files changed, 157 insertions(+), 68 deletions(-) diff --git a/crates/zakura-state/src/service/non_finalized_state.rs b/crates/zakura-state/src/service/non_finalized_state.rs index bfc3a8bcf6..f4787e9c98 100644 --- a/crates/zakura-state/src/service/non_finalized_state.rs +++ b/crates/zakura-state/src/service/non_finalized_state.rs @@ -39,14 +39,38 @@ mod tests; pub(crate) use backup::write_semantically_verified_backup_block; pub(crate) use chain::{Chain, SpendingTransactionId}; -macro_rules! record_contextual_duration { - ($metric_name:literal, $mined_metric_name:literal, $duration:expr, $is_mined:expr $(,)?) => {{ - let duration = $duration.as_secs_f64(); - metrics::histogram!($metric_name).record(duration); - if $is_mined { - metrics::histogram!($mined_metric_name).record(duration); +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ContextualMetrics { + Disabled, + AllBlocks, + Mined, +} + +impl ContextualMetrics { + pub(crate) fn for_commit(is_mined: bool) -> Self { + if is_mined { + Self::Mined + } else { + Self::AllBlocks } - }}; + } + + pub(crate) fn record_duration( + self, + metric_name: &'static str, + mined_metric_name: &'static str, + duration: std::time::Duration, + ) { + if self == Self::Disabled { + return; + } + + let duration = duration.as_secs_f64(); + metrics::histogram!(metric_name).record(duration); + if self == Self::Mined { + metrics::histogram!(mined_metric_name).record(duration); + } + } } /// The state of the chains in memory, including queued blocks. @@ -366,7 +390,7 @@ impl NonFinalizedState { prepared: SemanticallyVerifiedBlock, finalized_state: &ZakuraDb, ) -> Result<(), ValidateContextError> { - self.commit_block_with_metrics(prepared, finalized_state, false) + self.commit_block_with_metrics(prepared, finalized_state, ContextualMetrics::Disabled) } #[tracing::instrument( @@ -378,25 +402,24 @@ impl NonFinalizedState { &mut self, prepared: SemanticallyVerifiedBlock, finalized_state: &ZakuraDb, - is_mined: bool, + contextual_metrics: ContextualMetrics, ) -> Result<(), ValidateContextError> { let parent_hash = prepared.block.header.previous_block_hash; let (height, hash) = (prepared.height, prepared.hash); let parent_chain_start = Instant::now(); let parent_chain = self.parent_chain(parent_hash); - record_contextual_duration!( + contextual_metrics.record_duration( "state.contextual.parent_chain.duration_seconds", "state.contextual.mined.parent_chain.duration_seconds", parent_chain_start.elapsed(), - is_mined, ); let parent_chain = parent_chain?; // If the block is invalid, return the error, // and drop the cloned parent Arc, or newly created chain fork. let modified_chain = - self.validate_and_commit(parent_chain, prepared, finalized_state, is_mined)?; + self.validate_and_commit(parent_chain, prepared, finalized_state, contextual_metrics)?; // If the block is valid: // - add the new chain fork or updated chain to the set of recent chains @@ -555,7 +578,7 @@ impl NonFinalizedState { prepared: SemanticallyVerifiedBlock, finalized_state: &ZakuraDb, ) -> Result<(), ValidateContextError> { - self.commit_new_chain_with_metrics(prepared, finalized_state, false) + self.commit_new_chain_with_metrics(prepared, finalized_state, ContextualMetrics::Disabled) } #[tracing::instrument( @@ -568,7 +591,7 @@ impl NonFinalizedState { &mut self, prepared: SemanticallyVerifiedBlock, finalized_state: &ZakuraDb, - is_mined: bool, + contextual_metrics: ContextualMetrics, ) -> Result<(), ValidateContextError> { let chain_new_start = Instant::now(); let chain: Result = (|| { @@ -593,19 +616,22 @@ impl NonFinalizedState { finalized_state.finalized_value_pool(), )) })(); - record_contextual_duration!( + contextual_metrics.record_duration( "state.contextual.chain_new.duration_seconds", "state.contextual.mined.chain_new.duration_seconds", chain_new_start.elapsed(), - is_mined, ); let chain = chain?; let (height, hash) = (prepared.height, prepared.hash); // If the block is invalid, return the error, and drop the newly created chain fork - let chain = - self.validate_and_commit(Arc::new(chain), prepared, finalized_state, is_mined)?; + let chain = self.validate_and_commit( + Arc::new(chain), + prepared, + finalized_state, + contextual_metrics, + )?; // If the block is valid, add the new chain fork to the set of recent chains. self.insert(chain); @@ -625,7 +651,7 @@ impl NonFinalizedState { new_chain: Arc, prepared: SemanticallyVerifiedBlock, finalized_state: &ZakuraDb, - is_mined: bool, + contextual_metrics: ContextualMetrics, ) -> Result, ValidateContextError> { if self .invalidated_blocks @@ -642,11 +668,10 @@ impl NonFinalizedState { // TODO: if these disk reads show up in profiles, run them in parallel, using std::thread::spawn() let unspent_utxo_snapshot_start = Instant::now(); let unspent_utxos = new_chain.unspent_utxos(); - record_contextual_duration!( + contextual_metrics.record_duration( "state.contextual.unspent_utxo_snapshot.duration_seconds", "state.contextual.mined.unspent_utxo_snapshot.duration_seconds", unspent_utxo_snapshot_start.elapsed(), - is_mined, ); let transparent_spend_start = Instant::now(); @@ -656,11 +681,10 @@ impl NonFinalizedState { &new_chain.spent_utxos, finalized_state, ); - record_contextual_duration!( + contextual_metrics.record_duration( "state.contextual.transparent_spend.duration_seconds", "state.contextual.mined.transparent_spend.duration_seconds", transparent_spend_start.elapsed(), - is_mined, ); let spent_utxos = spent_utxos?; @@ -672,11 +696,10 @@ impl NonFinalizedState { &new_chain, &prepared, ); - record_contextual_duration!( + contextual_metrics.record_duration( "state.contextual.shielded_anchors.duration_seconds", "state.contextual.mined.shielded_anchors.duration_seconds", shielded_anchor_start.elapsed(), - is_mined, ); shielded_anchors?; @@ -687,11 +710,10 @@ impl NonFinalizedState { &new_chain, &prepared, ); - record_contextual_duration!( + contextual_metrics.record_duration( "state.contextual.sprout_anchor_fetch.duration_seconds", "state.contextual.mined.sprout_anchor_fetch.duration_seconds", sprout_anchor_fetch_start.elapsed(), - is_mined, ); // Quick check that doesn't read from disk @@ -710,11 +732,10 @@ impl NonFinalizedState { spent_utxo_count, }, ); - record_contextual_duration!( + contextual_metrics.record_duration( "state.contextual.block_construction.duration_seconds", "state.contextual.mined.block_construction.duration_seconds", contextual_block_start.elapsed(), - is_mined, ); let contextual = contextual?; @@ -723,13 +744,12 @@ impl NonFinalizedState { new_chain, contextual, sprout_final_treestates, - is_mined, + contextual_metrics, ); - record_contextual_duration!( + contextual_metrics.record_duration( "state.contextual.parallel_update.duration_seconds", "state.contextual.mined.parallel_update.duration_seconds", parallel_update_start.elapsed(), - is_mined, ); result } @@ -741,7 +761,7 @@ impl NonFinalizedState { new_chain: Arc, contextual: ContextuallyVerifiedBlock, sprout_final_treestates: HashMap>, - is_mined: bool, + contextual_metrics: ContextualMetrics, ) -> Result, ValidateContextError> { let mut block_commitment_result = None; let mut sprout_anchor_result = None; @@ -806,29 +826,25 @@ impl NonFinalizedState { // These task durations overlap. Only `parallel_update` measures their // combined critical-path wall time. - record_contextual_duration!( + contextual_metrics.record_duration( "state.contextual.parallel_task.block_commitment.duration_seconds", "state.contextual.mined.parallel_task.block_commitment.duration_seconds", block_commitment_duration, - is_mined, ); - record_contextual_duration!( + contextual_metrics.record_duration( "state.contextual.parallel_task.sprout_anchor_check.duration_seconds", "state.contextual.mined.parallel_task.sprout_anchor_check.duration_seconds", sprout_anchor_duration, - is_mined, ); - record_contextual_duration!( + contextual_metrics.record_duration( "state.contextual.parallel_task.chain_clone.duration_seconds", "state.contextual.mined.parallel_task.chain_clone.duration_seconds", chain_clone_duration, - is_mined, ); - record_contextual_duration!( + contextual_metrics.record_duration( "state.contextual.parallel_task.chain_push.duration_seconds", "state.contextual.mined.parallel_task.chain_push.duration_seconds", chain_push_duration, - is_mined, ); block_commitment_result?; diff --git a/crates/zakura-state/src/service/non_finalized_state/tests.rs b/crates/zakura-state/src/service/non_finalized_state/tests.rs index dd920a1960..7a90a67cf6 100644 --- a/crates/zakura-state/src/service/non_finalized_state/tests.rs +++ b/crates/zakura-state/src/service/non_finalized_state/tests.rs @@ -2,3 +2,82 @@ mod prop; mod vectors; + +use std::sync::Mutex; + +use metrics::{Counter, Gauge, Histogram, Key, KeyName, Metadata, Recorder, SharedString, Unit}; + +use super::ContextualMetrics; + +#[derive(Default)] +struct MetricNameRecorder { + histogram_names: Mutex>, +} + +impl Recorder for MetricNameRecorder { + fn describe_counter(&self, _key: KeyName, _unit: Option, _description: SharedString) {} + + fn describe_gauge(&self, _key: KeyName, _unit: Option, _description: SharedString) {} + + fn describe_histogram(&self, _key: KeyName, _unit: Option, _description: SharedString) {} + + fn register_counter(&self, _key: &Key, _metadata: &Metadata<'_>) -> Counter { + Counter::noop() + } + + fn register_gauge(&self, _key: &Key, _metadata: &Metadata<'_>) -> Gauge { + Gauge::noop() + } + + fn register_histogram(&self, key: &Key, _metadata: &Metadata<'_>) -> Histogram { + self.histogram_names + .lock() + .expect("the metric test does not poison its recorder") + .push(key.name().to_owned()); + Histogram::noop() + } +} + +#[test] +fn contextual_metrics_record_only_selected_series() { + let recorder = MetricNameRecorder::default(); + + assert_eq!( + ContextualMetrics::for_commit(false), + ContextualMetrics::AllBlocks + ); + assert_eq!( + ContextualMetrics::for_commit(true), + ContextualMetrics::Mined + ); + + metrics::with_local_recorder(&recorder, || { + ContextualMetrics::Disabled.record_duration( + "test.contextual.disabled.all", + "test.contextual.disabled.mined", + std::time::Duration::ZERO, + ); + ContextualMetrics::AllBlocks.record_duration( + "test.contextual.all.all", + "test.contextual.all.mined", + std::time::Duration::ZERO, + ); + ContextualMetrics::Mined.record_duration( + "test.contextual.mined.all", + "test.contextual.mined.mined", + std::time::Duration::ZERO, + ); + }); + + assert_eq!( + *recorder + .histogram_names + .lock() + .expect("the metric test does not poison its recorder"), + [ + "test.contextual.all.all", + "test.contextual.mined.all", + "test.contextual.mined.mined", + ] + ); +} diff --git a/crates/zakura-state/src/service/write.rs b/crates/zakura-state/src/service/write.rs index 32ef5a9380..234e6ce306 100644 --- a/crates/zakura-state/src/service/write.rs +++ b/crates/zakura-state/src/service/write.rs @@ -45,7 +45,7 @@ use crate::{ DiskWriteBatch, FinalizedState, VctAuthenticationProof, VctAuxiliaryFailureAttribution, VctAuxiliaryWindow, VctSuccessorWitness, ZakuraDb, }, - non_finalized_state::NonFinalizedState, + non_finalized_state::{ContextualMetrics, NonFinalizedState}, queued_blocks::{QueuedCheckpointVerified, QueuedSemanticallyVerified}, ChainTipBlock, ChainTipSender, InvalidateError, ReconsiderError, }, @@ -1260,16 +1260,6 @@ fn commit_operator_change( /// We allow enough space for multiple concurrent chain forks with errors. const REJECTED_ANCESTOR_MAP_LIMIT: usize = MAX_BLOCK_REORG_HEIGHT as usize * 2; -macro_rules! record_contextual_duration { - ($metric_name:literal, $mined_metric_name:literal, $duration:expr, $is_mined:expr $(,)?) => {{ - let duration = $duration.as_secs_f64(); - metrics::histogram!($metric_name).record(duration); - if $is_mined { - metrics::histogram!($mined_metric_name).record(duration); - } - }}; -} - /// Run contextual validation on the prepared block and add it to the /// non-finalized state if it is contextually valid. pub(crate) fn validate_and_commit_non_finalized( @@ -1281,7 +1271,7 @@ pub(crate) fn validate_and_commit_non_finalized( finalized_state, non_finalized_state, prepared, - false, + ContextualMetrics::Disabled, ) } @@ -1299,32 +1289,38 @@ fn validate_and_commit_non_finalized_with_metrics( finalized_state: &ZakuraDb, non_finalized_state: &mut NonFinalizedState, prepared: SemanticallyVerifiedBlock, - is_mined: bool, + contextual_metrics: ContextualMetrics, ) -> Result<(), ValidateContextError> { let total_start = Instant::now(); let initial_checks_start = Instant::now(); let initial_checks = check::initial_contextual_validity(finalized_state, non_finalized_state, &prepared); - record_contextual_duration!( + contextual_metrics.record_duration( "state.contextual.initial_checks.duration_seconds", "state.contextual.mined.initial_checks.duration_seconds", initial_checks_start.elapsed(), - is_mined, ); let result = initial_checks.and_then(|()| { let parent_hash = prepared.block.header.previous_block_hash; if finalized_state.finalized_tip_hash() == parent_hash { - non_finalized_state.commit_new_chain_with_metrics(prepared, finalized_state, is_mined) + non_finalized_state.commit_new_chain_with_metrics( + prepared, + finalized_state, + contextual_metrics, + ) } else { - non_finalized_state.commit_block_with_metrics(prepared, finalized_state, is_mined) + non_finalized_state.commit_block_with_metrics( + prepared, + finalized_state, + contextual_metrics, + ) } }); - record_contextual_duration!( + contextual_metrics.record_duration( "state.contextual.total.duration_seconds", "state.contextual.mined.total.duration_seconds", total_start.elapsed(), - is_mined, ); result @@ -2561,6 +2557,7 @@ impl WriteBlockWorkerTask { metrics::histogram!("state.block_writer.queue.duration_seconds") .record(writer_queue_duration); let is_mined = admission.is_some(); + let contextual_metrics = ContextualMetrics::for_commit(is_mined); if is_mined { metrics::histogram!("state.block_writer.queue.mined.duration_seconds") .record(writer_queue_duration); @@ -2583,17 +2580,16 @@ impl WriteBlockWorkerTask { if let Some(writer) = header_chain.as_ref() { let snapshot_clone_start = Instant::now(); let mut staged = non_finalized_state.clone(); - record_contextual_duration!( + contextual_metrics.record_duration( "state.contextual.snapshot_clone.duration_seconds", "state.contextual.mined.snapshot_clone.duration_seconds", snapshot_clone_start.elapsed(), - is_mined, ); validate_and_commit_non_finalized_with_metrics( &finalized_state.db, &mut staged, queued_child, - is_mined, + contextual_metrics, ) .map_err(|error| CommitBlockError::from(Box::new(error))) .and_then(|()| { @@ -2624,11 +2620,10 @@ impl WriteBlockWorkerTask { } }) }); - record_contextual_duration!( + contextual_metrics.record_duration( "state.contextual.header_transition_prepare.duration_seconds", "state.contextual.mined.header_transition_prepare.duration_seconds", transition_prepare_start.elapsed(), - is_mined, ); let transition = transition?; @@ -2639,11 +2634,10 @@ impl WriteBlockWorkerTask { .map_err(|error| CommitBlockError::HeaderChainError { error: error.to_string(), }); - record_contextual_duration!( + contextual_metrics.record_duration( "state.contextual.header_transition_commit.duration_seconds", "state.contextual.mined.header_transition_commit.duration_seconds", transition_commit_start.elapsed(), - is_mined, ); result }) @@ -2652,7 +2646,7 @@ impl WriteBlockWorkerTask { &finalized_state.db, non_finalized_state, queued_child, - is_mined, + contextual_metrics, ) .map_err(|error| CommitBlockError::from(Box::new(error))) } From 058c09fd7f1033238475aec7a3e82631cdba9231 Mon Sep 17 00:00:00 2001 From: evan-forbes Date: Mon, 24 Aug 2026 18:53:03 -0500 Subject: [PATCH 18/22] fix(mining): isolate server templates from proposals --- crates/zakura-consensus/src/block.rs | 12 +- crates/zakura-consensus/src/block/prepared.rs | 480 ++++++++++++++++-- crates/zakura-consensus/src/block/request.rs | 19 + crates/zakura-consensus/src/block/tests.rs | 61 +++ crates/zakura-consensus/src/lib.rs | 5 +- crates/zakura-rpc/src/methods.rs | 1 + .../zakura-rpc/src/methods/tests/snapshot.rs | 26 +- .../src/methods/types/get_block_template.rs | 1 + docs/changelog/params.md | 6 +- 9 files changed, 572 insertions(+), 39 deletions(-) diff --git a/crates/zakura-consensus/src/block.rs b/crates/zakura-consensus/src/block.rs index b89ff99883..bc3ebbffb3 100644 --- a/crates/zakura-consensus/src/block.rs +++ b/crates/zakura-consensus/src/block.rs @@ -38,7 +38,7 @@ mod prepared; pub mod request; pub mod subsidy; -pub use request::Request; +pub use request::{PreparedCandidateSource, Request}; #[cfg(test)] mod tests; @@ -514,7 +514,15 @@ where }; if let (Ok(_), Some(cache_copy)) = (&response, cache_copy) { let candidate = cache_copy.block.clone(); - prepared_candidates.insert(&candidate, request.work_id(), cache_copy, &network); + prepared_candidates.insert( + &candidate, + request.work_id(), + request + .prepared_candidate_source() + .expect("cached preparation has a candidate source"), + cache_copy, + &network, + ); metrics::histogram!("mining.preparation.duration_seconds").record( preparation_start .expect("cached preparation records its start time") diff --git a/crates/zakura-consensus/src/block/prepared.rs b/crates/zakura-consensus/src/block/prepared.rs index 35cfce0f6c..e78ce5448a 100644 --- a/crates/zakura-consensus/src/block/prepared.rs +++ b/crates/zakura-consensus/src/block/prepared.rs @@ -14,8 +14,12 @@ use zakura_chain::{ }; use zakura_state::SemanticallyVerifiedBlock; -const MAX_ENTRIES: usize = 32; -const MAX_BYTES: usize = 64 * 1024 * 1024; +use super::PreparedCandidateSource; + +const SERVER_MAX_ENTRIES: usize = 24; +const SERVER_MAX_BYTES: usize = 48 * 1024 * 1024; +const PROPOSAL_MAX_ENTRIES: usize = 8; +const PROPOSAL_MAX_BYTES: usize = 16 * 1024 * 1024; const ENTRY_TTL: Duration = Duration::from_secs(10 * 60); #[derive(Clone, Default)] @@ -28,14 +32,22 @@ impl std::fmt::Debug for PreparedCandidateCache { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); f.debug_struct("PreparedCandidateCache") - .field("entries", &inner.entries.len()) - .field("bytes", &inner.bytes) + .field("server_entries", &inner.server.entries.len()) + .field("server_bytes", &inner.server.bytes) + .field("proposal_entries", &inner.proposals.entries.len()) + .field("proposal_bytes", &inner.proposals.bytes) .finish() } } #[derive(Default)] struct CacheInner { + server: Partition, + proposals: Partition, +} + +#[derive(Default)] +struct Partition { entries: VecDeque, bytes: usize, } @@ -64,7 +76,7 @@ impl PreparedCandidateCache { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); inner.prune_expired(); - if inner.entries.is_empty() { + if inner.is_empty() { metrics::counter!("mining.prepared_cache.misses").increment(1); return None; } @@ -79,11 +91,7 @@ impl PreparedCandidateCache { inner.prune_expired(); if let Some(work_id) = work_id { - if let Some(entry) = inner - .entries - .iter() - .find(|entry| entry.work_id.as_deref() == Some(work_id)) - { + if let Some(entry) = inner.find_work_id(work_id) { if entry.immutable_bytes == immutable_bytes { metrics::counter!("mining.prepared_cache.hits").increment(1); return Some(entry.prepared.clone()); @@ -94,9 +102,7 @@ impl PreparedCandidateCache { } } - if let Some(entry) = inner.entries.iter().find(|entry| { - entry.fingerprint == fingerprint && entry.immutable_bytes == immutable_bytes - }) { + if let Some(entry) = inner.find_candidate(fingerprint, &immutable_bytes) { metrics::counter!("mining.prepared_cache.hits").increment(1); return Some(entry.prepared.clone()); } @@ -109,6 +115,7 @@ impl PreparedCandidateCache { &self, block: &Block, work_id: Option<&str>, + source: PreparedCandidateSource, prepared: SemanticallyVerifiedBlock, network: &Network, ) { @@ -120,17 +127,29 @@ impl PreparedCandidateCache { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); inner.prune_expired(); - if work_id.is_some_and(|work_id| { - inner.entries.iter().any(|entry| { - entry.work_id.as_deref() == Some(work_id) - && entry.immutable_bytes != immutable_bytes - }) - }) { - metrics::counter!("mining.prepared_cache.work_id_conflicts").increment(1); - return; + + if let Some(work_id) = work_id { + if let Some((existing_source, entry)) = inner.find_work_id_with_source(work_id) { + if entry.immutable_bytes == immutable_bytes { + return; + } + + metrics::counter!( + "mining.prepared_cache.work_id_conflicts", + "source" => source.metric_label() + ) + .increment(1); + let server_reclaims_proposal = source == PreparedCandidateSource::ServerTemplate + && existing_source == PreparedCandidateSource::ClientProposal; + if !server_reclaims_proposal { + return; + } + } } + + let partition = inner.partition(source); let existing_work_id = if work_id.is_none() { - inner + partition .entries .iter() .find(|entry| { @@ -147,20 +166,37 @@ impl PreparedCandidateCache { .or_else(|| existing_work_id.as_ref().map(String::len)) .unwrap_or(0), ); - if size > MAX_BYTES { + let (max_entries, max_bytes) = source.limits(); + if size > max_bytes { return; } let work_id = work_id.map(ToOwned::to_owned).or(existing_work_id); - inner.remove_matching(work_id.as_deref(), fingerprint, &immutable_bytes); - while inner.entries.len() >= MAX_ENTRIES || inner.bytes.saturating_add(size) > MAX_BYTES { - if !inner.evict_oldest() { + if let Some(work_id) = work_id.as_deref() { + if source == PreparedCandidateSource::ServerTemplate + && inner + .proposals + .entries + .iter() + .any(|entry| entry.work_id.as_deref() == Some(work_id)) + { + inner.proposals.remove_work_id(work_id); + } + } + + let partition = inner.partition(source); + partition.remove_matching(work_id.as_deref(), fingerprint, &immutable_bytes); + + while partition.entries.len() >= max_entries + || partition.bytes.saturating_add(size) > max_bytes + { + if !partition.evict_oldest(source) { break; } } - inner.bytes = inner.bytes.saturating_add(size); - inner.entries.push_back(Entry { + partition.bytes = partition.bytes.saturating_add(size); + partition.entries.push_back(Entry { work_id, fingerprint, immutable_bytes, @@ -173,13 +209,66 @@ impl PreparedCandidateCache { impl CacheInner { fn prune_expired(&mut self) { + self.server + .prune_expired(PreparedCandidateSource::ServerTemplate); + self.proposals + .prune_expired(PreparedCandidateSource::ClientProposal); + } + + fn is_empty(&self) -> bool { + self.server.entries.is_empty() && self.proposals.entries.is_empty() + } + + fn partition(&mut self, source: PreparedCandidateSource) -> &mut Partition { + match source { + PreparedCandidateSource::ServerTemplate => &mut self.server, + PreparedCandidateSource::ClientProposal => &mut self.proposals, + } + } + + fn find_work_id(&self, work_id: &str) -> Option<&Entry> { + self.server + .entries + .iter() + .chain(self.proposals.entries.iter()) + .find(|entry| entry.work_id.as_deref() == Some(work_id)) + } + + fn find_work_id_with_source(&self, work_id: &str) -> Option<(PreparedCandidateSource, &Entry)> { + self.server + .entries + .iter() + .find(|entry| entry.work_id.as_deref() == Some(work_id)) + .map(|entry| (PreparedCandidateSource::ServerTemplate, entry)) + .or_else(|| { + self.proposals + .entries + .iter() + .find(|entry| entry.work_id.as_deref() == Some(work_id)) + .map(|entry| (PreparedCandidateSource::ClientProposal, entry)) + }) + } + + fn find_candidate(&self, fingerprint: [u8; 32], immutable_bytes: &[u8]) -> Option<&Entry> { + self.server + .entries + .iter() + .chain(self.proposals.entries.iter()) + .find(|entry| { + entry.fingerprint == fingerprint && entry.immutable_bytes == immutable_bytes + }) + } +} + +impl Partition { + fn prune_expired(&mut self, source: PreparedCandidateSource) { let now = Instant::now(); while self .entries .front() .is_some_and(|entry| entry.expires_at <= now) { - self.evict_oldest(); + self.evict_oldest(source); } } @@ -201,16 +290,51 @@ impl CacheInner { } } - fn evict_oldest(&mut self) -> bool { + fn remove_work_id(&mut self, work_id: &str) { + let Some(index) = self + .entries + .iter() + .position(|entry| entry.work_id.as_deref() == Some(work_id)) + else { + return; + }; + let entry = self + .entries + .remove(index) + .expect("entry exists because its index came from the same deque"); + self.bytes = self.bytes.saturating_sub(entry.size); + } + + fn evict_oldest(&mut self, source: PreparedCandidateSource) -> bool { let Some(entry) = self.entries.pop_front() else { return false; }; self.bytes = self.bytes.saturating_sub(entry.size); - metrics::counter!("mining.prepared_cache.evictions").increment(1); + metrics::counter!( + "mining.prepared_cache.evictions", + "source" => source.metric_label() + ) + .increment(1); true } } +impl PreparedCandidateSource { + fn limits(self) -> (usize, usize) { + match self { + Self::ServerTemplate => (SERVER_MAX_ENTRIES, SERVER_MAX_BYTES), + Self::ClientProposal => (PROPOSAL_MAX_ENTRIES, PROPOSAL_MAX_BYTES), + } + } + + fn metric_label(self) -> &'static str { + match self { + Self::ServerTemplate => "server_template", + Self::ClientProposal => "client_proposal", + } + } +} + fn immutable_candidate_bytes(block: &Block, network: &Network) -> Vec { let mut header: Header = *block.header; header.time = @@ -245,13 +369,41 @@ mod tests { .expect("the genesis test vector is valid") } + fn distinct_block(index: u8) -> Block { + let mut block = test_block(); + Arc::make_mut(&mut block.header).previous_block_hash = Hash([index; 32]); + block + } + + fn insert( + cache: &PreparedCandidateCache, + block: &Block, + work_id: &str, + source: PreparedCandidateSource, + network: &Network, + ) { + cache.insert( + block, + Some(work_id), + source, + SemanticallyVerifiedBlock::from(Arc::new(block.clone())), + network, + ); + } + #[test] fn solved_header_fields_reuse_prepared_candidate() { let network = Network::Mainnet; let original = test_block(); let prepared = SemanticallyVerifiedBlock::from(Arc::new(original.clone())); let cache = PreparedCandidateCache::default(); - cache.insert(&original, Some("work"), prepared, &network); + cache.insert( + &original, + Some("work"), + PreparedCandidateSource::ServerTemplate, + prepared, + &network, + ); let mut solved = original; let header = Arc::make_mut(&mut solved.header); @@ -269,7 +421,13 @@ mod tests { let original = test_block(); let prepared = SemanticallyVerifiedBlock::from(Arc::new(original.clone())); let cache = PreparedCandidateCache::default(); - cache.insert(&original, Some("work"), prepared, &network); + cache.insert( + &original, + Some("work"), + PreparedCandidateSource::ServerTemplate, + prepared, + &network, + ); let mut changed_parent = original.clone(); Arc::make_mut(&mut changed_parent.header).previous_block_hash = Hash([1; 32]); @@ -322,12 +480,14 @@ mod tests { cache.insert( &original, Some("work"), + PreparedCandidateSource::ServerTemplate, SemanticallyVerifiedBlock::from(Arc::new(original.clone())), &network, ); cache.insert( &replacement, Some("work"), + PreparedCandidateSource::ClientProposal, SemanticallyVerifiedBlock::from(Arc::new(replacement.clone())), &network, ); @@ -339,9 +499,263 @@ mod tests { .0 .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) + .server .entries .len(), 1 ); } + + #[test] + fn proposal_eviction_does_not_evict_server_candidates() { + let network = Network::Mainnet; + let cache = PreparedCandidateCache::default(); + let server = distinct_block(100); + insert( + &cache, + &server, + "server", + PreparedCandidateSource::ServerTemplate, + &network, + ); + + let proposals: Vec<_> = (0..=PROPOSAL_MAX_ENTRIES) + .map(|index| distinct_block(index as u8)) + .collect(); + for (index, proposal) in proposals.iter().enumerate() { + insert( + &cache, + proposal, + &format!("proposal-{index}"), + PreparedCandidateSource::ClientProposal, + &network, + ); + } + + assert!(cache + .lookup(&proposals[0], Some("proposal-0"), &network) + .is_none()); + assert!(cache + .lookup(&proposals[1], Some("proposal-1"), &network) + .is_some()); + assert!(cache.lookup(&server, Some("server"), &network).is_some()); + + let inner = cache + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + assert_eq!(inner.server.entries.len(), 1); + assert_eq!(inner.proposals.entries.len(), PROPOSAL_MAX_ENTRIES); + assert!(inner.server.bytes <= SERVER_MAX_BYTES); + assert!(inner.proposals.bytes <= PROPOSAL_MAX_BYTES); + } + + #[test] + fn server_eviction_does_not_evict_proposals() { + let network = Network::Mainnet; + let cache = PreparedCandidateCache::default(); + let proposal = distinct_block(100); + insert( + &cache, + &proposal, + "proposal", + PreparedCandidateSource::ClientProposal, + &network, + ); + + let servers: Vec<_> = (0..=SERVER_MAX_ENTRIES) + .map(|index| distinct_block(index as u8)) + .collect(); + for (index, server) in servers.iter().enumerate() { + insert( + &cache, + server, + &format!("server-{index}"), + PreparedCandidateSource::ServerTemplate, + &network, + ); + } + + assert!(cache + .lookup(&servers[0], Some("server-0"), &network) + .is_none()); + assert!(cache + .lookup(&servers[1], Some("server-1"), &network) + .is_some()); + assert!(cache + .lookup(&proposal, Some("proposal"), &network) + .is_some()); + + let inner = cache + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + assert_eq!(inner.server.entries.len(), SERVER_MAX_ENTRIES); + assert_eq!(inner.proposals.entries.len(), 1); + assert!(inner.server.bytes <= SERVER_MAX_BYTES); + assert!(inner.proposals.bytes <= PROPOSAL_MAX_BYTES); + } + + #[test] + fn proposal_cannot_replace_a_server_work_id() { + let network = Network::Mainnet; + let cache = PreparedCandidateCache::default(); + let server = distinct_block(1); + let proposal = distinct_block(2); + insert( + &cache, + &server, + "shared", + PreparedCandidateSource::ServerTemplate, + &network, + ); + insert( + &cache, + &proposal, + "shared", + PreparedCandidateSource::ClientProposal, + &network, + ); + + assert!(cache.lookup(&server, Some("shared"), &network).is_some()); + assert!(cache.lookup(&proposal, Some("shared"), &network).is_none()); + let inner = cache + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + assert_eq!(inner.server.entries.len(), 1); + assert!(inner.proposals.entries.is_empty()); + } + + #[test] + fn server_reclaims_its_work_id_from_a_proposal() { + let network = Network::Mainnet; + let cache = PreparedCandidateCache::default(); + let proposal = distinct_block(1); + let server = distinct_block(2); + insert( + &cache, + &proposal, + "shared", + PreparedCandidateSource::ClientProposal, + &network, + ); + insert( + &cache, + &server, + "shared", + PreparedCandidateSource::ServerTemplate, + &network, + ); + + assert!(cache.lookup(&server, Some("shared"), &network).is_some()); + assert!(cache.lookup(&proposal, Some("shared"), &network).is_none()); + let inner = cache + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + assert_eq!(inner.server.entries.len(), 1); + assert!(inner.proposals.entries.is_empty()); + } + + #[test] + fn each_source_retains_the_same_candidate_under_its_own_work_id() { + let network = Network::Mainnet; + let cache = PreparedCandidateCache::default(); + let candidate = test_block(); + insert( + &cache, + &candidate, + "server", + PreparedCandidateSource::ServerTemplate, + &network, + ); + insert( + &cache, + &candidate, + "proposal", + PreparedCandidateSource::ClientProposal, + &network, + ); + + assert!(cache.lookup(&candidate, Some("server"), &network).is_some()); + assert!(cache + .lookup(&candidate, Some("proposal"), &network) + .is_some()); + let inner = cache + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + assert_eq!(inner.server.entries.len(), 1); + assert_eq!(inner.proposals.entries.len(), 1); + } + + #[test] + fn identical_work_id_and_candidate_leave_the_existing_mapping_unchanged() { + let network = Network::Mainnet; + let cache = PreparedCandidateCache::default(); + let candidate = test_block(); + insert( + &cache, + &candidate, + "shared", + PreparedCandidateSource::ClientProposal, + &network, + ); + let original_expiration = cache + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .proposals + .entries[0] + .expires_at; + + insert( + &cache, + &candidate, + "shared", + PreparedCandidateSource::ServerTemplate, + &network, + ); + + let inner = cache + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + assert!(inner.server.entries.is_empty()); + assert_eq!(inner.proposals.entries.len(), 1); + assert_eq!(inner.proposals.entries[0].expires_at, original_expiration); + } + + #[test] + fn oversized_entries_do_not_exceed_partition_byte_limits() { + let network = Network::Mainnet; + let cache = PreparedCandidateCache::default(); + let candidate = test_block(); + let oversized_proposal_id = "p".repeat(PROPOSAL_MAX_BYTES); + insert( + &cache, + &candidate, + &oversized_proposal_id, + PreparedCandidateSource::ClientProposal, + &network, + ); + let oversized_server_id = "s".repeat(SERVER_MAX_BYTES); + insert( + &cache, + &candidate, + &oversized_server_id, + PreparedCandidateSource::ServerTemplate, + &network, + ); + + let inner = cache + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + assert!(inner.server.entries.is_empty()); + assert!(inner.proposals.entries.is_empty()); + assert_eq!(inner.server.bytes, 0); + assert_eq!(inner.proposals.bytes, 0); + } } diff --git a/crates/zakura-consensus/src/block/request.rs b/crates/zakura-consensus/src/block/request.rs index 780a3e7dfe..474e147f43 100644 --- a/crates/zakura-consensus/src/block/request.rs +++ b/crates/zakura-consensus/src/block/request.rs @@ -5,6 +5,15 @@ use std::sync::Arc; use zakura_chain::block::Block; use zakura_state::BlockAdmission; +/// Identifies who supplied a prepared mining candidate. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PreparedCandidateSource { + /// The node's background `getblocktemplate` preparation. + ServerTemplate, + /// A client's proposal-mode `getblocktemplate` request. + ClientProposal, +} + #[derive(Debug, Clone, PartialEq, Eq)] /// A request to the chain or block verifier pub enum Request { @@ -29,6 +38,8 @@ pub enum Request { block: Arc, /// The template work ID, when one was assigned. work_id: Option, + /// The source that supplied the candidate. + source: PreparedCandidateSource, }, } @@ -56,6 +67,14 @@ impl Request { matches!(self, Request::Prepare { .. }) } + /// Returns the prepared candidate source. + pub fn prepared_candidate_source(&self) -> Option { + match self { + Request::Prepare { source, .. } => Some(*source), + _ => None, + } + } + /// Returns the supplied mining work ID. pub fn work_id(&self) -> Option<&str> { match self { diff --git a/crates/zakura-consensus/src/block/tests.rs b/crates/zakura-consensus/src/block/tests.rs index 5c0b963d43..78331b571a 100644 --- a/crates/zakura-consensus/src/block/tests.rs +++ b/crates/zakura-consensus/src/block/tests.rs @@ -175,6 +175,7 @@ where .call(Request::Prepare { block, work_id: Some("work".to_owned()), + source: PreparedCandidateSource::ServerTemplate, }) .await .expect("the candidate prepares successfully"); @@ -345,6 +346,7 @@ async fn failed_preparation_does_not_populate_the_cache() { .call(Request::Prepare { block: candidate.clone(), work_id: Some("work".to_owned()), + source: PreparedCandidateSource::ServerTemplate, }) .await; assert!(matches!( @@ -367,6 +369,65 @@ async fn failed_preparation_does_not_populate_the_cache() { assert_eq!(transaction_calls.load(Ordering::Relaxed), 2); } +#[tokio::test] +async fn proposal_validation_succeeds_when_cache_insertion_conflicts() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + let _init_guard = zakura_test::init(); + let network = librustzcash_conversion_test_network(NetworkUpgrade::Nu5); + let candidate = Arc::new(nu5_prepared_test_block(&network, None)); + let mut conflicting_proposal = (*candidate).clone(); + Arc::make_mut(&mut conflicting_proposal.header).previous_block_hash = block::Hash([1; 32]); + let conflicting_proposal = Arc::new(conflicting_proposal); + let transaction_calls = Arc::new(AtomicUsize::new(0)); + let transaction = service_fn({ + let transaction_calls = transaction_calls.clone(); + move |request| { + transaction_calls.fetch_add(1, Ordering::Relaxed); + async move { Ok::<_, BoxError>(accept_block_transaction(request)) } + } + }); + let state = service_fn(|request: zs::Request| async move { + let response = match request { + zs::Request::KnownBlock(_) => zs::Response::KnownBlock(None), + zs::Request::CheckBlockProposalValidity(_) => zs::Response::ValidBlockProposal, + zs::Request::CommitSemanticallyVerifiedBlockWithAdmission { block, .. } => { + zs::Response::Committed(block.hash) + } + _ => panic!("cache-conflict test received an unexpected request: {request:?}"), + }; + Ok::<_, BoxError>(response) + }); + let mut verifier = SemanticBlockVerifier::new(&network, state, transaction); + + prepare_for_test(&mut verifier, candidate).await; + let proposal_result = verifier + .ready() + .await + .expect("the verifier is ready") + .call(Request::Prepare { + block: conflicting_proposal.clone(), + work_id: Some("work".to_owned()), + source: PreparedCandidateSource::ClientProposal, + }) + .await; + assert!(proposal_result.is_ok()); + assert_eq!(transaction_calls.load(Ordering::Relaxed), 2); + + let commit_result = verifier + .ready() + .await + .expect("the verifier is ready") + .call(Request::CommitMined { + block: conflicting_proposal, + work_id: Some("work".to_owned()), + admission: zs::BlockAdmission::pending(), + }) + .await; + assert!(commit_result.is_ok()); + assert_eq!(transaction_calls.load(Ordering::Relaxed), 3); +} + // TODO: enable this test after implementing contextual verification // #[tokio::test] // #[ignore] diff --git a/crates/zakura-consensus/src/lib.rs b/crates/zakura-consensus/src/lib.rs index 53a9881a17..7c66e7080b 100644 --- a/crates/zakura-consensus/src/lib.rs +++ b/crates/zakura-consensus/src/lib.rs @@ -52,7 +52,10 @@ pub use block::check::difficulty_is_valid; #[cfg(any(test, feature = "proptest-impl"))] pub use checkpoint::CheckpointVerifier; -pub use block::{subsidy::funding_stream_address, Request, VerifyBlockError, MAX_BLOCK_SIGOPS}; +pub use block::{ + subsidy::funding_stream_address, PreparedCandidateSource, Request, VerifyBlockError, + MAX_BLOCK_SIGOPS, +}; pub use checkpoint::{VerifyCheckpointError, MAX_CHECKPOINT_BYTE_COUNT, MAX_CHECKPOINT_HEIGHT_GAP}; pub use config::Config; pub use error::BlockError; diff --git a/crates/zakura-rpc/src/methods.rs b/crates/zakura-rpc/src/methods.rs index 7f3d0727cf..3b19505501 100644 --- a/crates/zakura-rpc/src/methods.rs +++ b/crates/zakura-rpc/src/methods.rs @@ -1063,6 +1063,7 @@ where let request = zakura_consensus::Request::Prepare { block: Arc::new(block), work_id: Some(template.work_id().clone()), + source: zakura_consensus::PreparedCandidateSource::ServerTemplate, }; if let Err(error) = verifier.oneshot(request).await { tracing::debug!(?error, "background mining candidate preparation failed"); diff --git a/crates/zakura-rpc/src/methods/tests/snapshot.rs b/crates/zakura-rpc/src/methods/tests/snapshot.rs index 207eca66d3..e4fef6171a 100644 --- a/crates/zakura-rpc/src/methods/tests/snapshot.rs +++ b/crates/zakura-rpc/src/methods/tests/snapshot.rs @@ -1257,6 +1257,7 @@ pub async fn test_mining_rpcs( .as_ref() .zcash_deserialize_into() .expect("coinbase bytes are valid"); + let server_template = get_block_template.clone(); snapshot_rpc_getblocktemplate( "basic", @@ -1346,6 +1347,21 @@ pub async fn test_mining_rpcs( None, ); + let mut server_preparation_verifier = mock_block_verifier_router.clone(); + rpc_mock_state_verifier.prepare_template_in_background(&server_template); + server_preparation_verifier + .expect_request_that(|request| { + matches!( + request, + zakura_consensus::Request::Prepare { + source: zakura_consensus::PreparedCandidateSource::ServerTemplate, + .. + } + ) + }) + .await + .respond(Hash::from([0; 32])); + let get_block_template_fut = rpc_mock_state_verifier.get_block_template(Some(GetBlockTemplateParameters { mode: GetBlockTemplateRequestMode::Proposal, @@ -1355,7 +1371,15 @@ pub async fn test_mining_rpcs( let mock_block_verifier_router_request_handler = async move { mock_block_verifier_router - .expect_request_that(|req| matches!(req, zakura_consensus::Request::Prepare { .. })) + .expect_request_that(|request| { + matches!( + request, + zakura_consensus::Request::Prepare { + source: zakura_consensus::PreparedCandidateSource::ClientProposal, + .. + } + ) + }) .await .respond(Hash::from([0; 32])); }; diff --git a/crates/zakura-rpc/src/methods/types/get_block_template.rs b/crates/zakura-rpc/src/methods/types/get_block_template.rs index 3b90424f65..09de544dc1 100644 --- a/crates/zakura-rpc/src/methods/types/get_block_template.rs +++ b/crates/zakura-rpc/src/methods/types/get_block_template.rs @@ -781,6 +781,7 @@ where .call(zakura_consensus::Request::Prepare { block: Arc::new(block), work_id, + source: zakura_consensus::PreparedCandidateSource::ClientProposal, }) .await; diff --git a/docs/changelog/params.md b/docs/changelog/params.md index cb2c116bc6..705dc23159 100644 --- a/docs/changelog/params.md +++ b/docs/changelog/params.md @@ -33,8 +33,10 @@ Keep entries **newest-first**. Each row records: | `MINED_BLOCK_EVENT_SEND_TIMEOUT` | `crates/zakura-rpc/src/methods.rs` | new → `5 s` | [#748](https://github.com/zakura-core/zakura/pull/748) | Bound detached final gossip sends when the mined-block event channel stalls. | | `MAX_BACKGROUND_TEMPLATE_PREPARATIONS` | `crates/zakura-rpc/src/methods/types/get_block_template.rs` | new → `1` task | [#748](https://github.com/zakura-core/zakura/pull/748) | Prevent repeated template requests from delaying solved block submissions with concurrent preparation work. | | `ENTRY_TTL` | `crates/zakura-consensus/src/block/prepared.rs` | new → `10 min` | [#748](https://github.com/zakura-core/zakura/pull/748) | Expire prepared candidates after miners should have replaced their work. | -| `MAX_BYTES` | `crates/zakura-consensus/src/block/prepared.rs` | new → `64 MiB` | [#748](https://github.com/zakura-core/zakura/pull/748) | Bound prepared-candidate memory separately from the entry count. | -| `MAX_ENTRIES` | `crates/zakura-consensus/src/block/prepared.rs` | new → `32` candidates | [#748](https://github.com/zakura-core/zakura/pull/748) | Retain recent mining candidates while bounding verification-cache work and memory. | +| `SERVER_MAX_BYTES` | `crates/zakura-consensus/src/block/prepared.rs` | new → `48 MiB` | [#748](https://github.com/zakura-core/zakura/pull/748) | Bound server-template candidate memory within the existing 64 MiB total. | +| `SERVER_MAX_ENTRIES` | `crates/zakura-consensus/src/block/prepared.rs` | new → `24` candidates | [#748](https://github.com/zakura-core/zakura/pull/748) | Keep client proposals from evicting server-template candidates. | +| `PROPOSAL_MAX_BYTES` | `crates/zakura-consensus/src/block/prepared.rs` | new → `16 MiB` | [#748](https://github.com/zakura-core/zakura/pull/748) | Bound client-proposal candidate memory within the existing 64 MiB total. | +| `PROPOSAL_MAX_ENTRIES` | `crates/zakura-consensus/src/block/prepared.rs` | new → `8` candidates | [#748](https://github.com/zakura-core/zakura/pull/748) | Keep client proposals in an independent candidate partition. | | `MAX_PENDING_BLOCK_WAITS` | `crates/zakura-rpc/src/methods/types/submit_block.rs` | new → `32` waits | [#748](https://github.com/zakura-core/zakura/pull/748) | Reserve inbound capacity when peers request early-advertised blocks. | | `MAX_PENDING_BLOCKS` | `crates/zakura-rpc/src/methods/types/submit_block.rs` | new → `16` blocks | [#748](https://github.com/zakura-core/zakura/pull/748) | Bound mined blocks waiting for contextual commit. | | `PENDING_BLOCK_WAIT` | `crates/zakura-rpc/src/methods/types/submit_block.rs` | new → `15 s` | [#748](https://github.com/zakura-core/zakura/pull/748) | Let peers wait for early-advertised blocks below the legacy peer request timeout. | From cb43cccc5e36d41759ce0be0f13549f0a9007498 Mon Sep 17 00:00:00 2001 From: evan-forbes Date: Thu, 27 Aug 2026 11:52:40 -0500 Subject: [PATCH 19/22] fix(mining): harden optimistic block inventory --- crates/zakura-consensus/src/block.rs | 29 +-- crates/zakura-consensus/src/block/prepared.rs | 199 +++++++++++++----- crates/zakura-consensus/src/block/tests.rs | 2 +- crates/zakura-network/src/peer/connection.rs | 4 +- crates/zakura-network/src/peer_set/set.rs | 115 +++++----- .../src/peer_set/set/tests/vectors.rs | 125 +++++++---- crates/zakura-rpc/src/lib.rs | 2 +- crates/zakura-rpc/src/methods.rs | 108 ++++------ .../src/methods/types/get_block_template.rs | 77 +++++-- .../methods/types/get_block_template/tests.rs | 20 +- .../src/methods/types/submit_block.rs | 136 +++++++----- crates/zakura-state/src/error.rs | 7 + crates/zakura-state/src/request.rs | 21 +- crates/zakura-state/src/service.rs | 63 ++++-- .../zakura-state/src/service/queued_blocks.rs | 38 ++++ .../service/queued_blocks/tests/vectors.rs | 42 +++- .../components/inbound/tests/real_peer_set.rs | 2 - crates/zakurad/src/components/sync/gossip.rs | 130 ++++-------- .../src/components/sync/tests/gossip.rs | 20 +- crates/zakurad/tests/acceptance.rs | 9 +- 20 files changed, 696 insertions(+), 453 deletions(-) diff --git a/crates/zakura-consensus/src/block.rs b/crates/zakura-consensus/src/block.rs index bc3ebbffb3..47ac956cec 100644 --- a/crates/zakura-consensus/src/block.rs +++ b/crates/zakura-consensus/src/block.rs @@ -302,8 +302,10 @@ where if request.is_mined_commit() { let solved_header_start = std::time::Instant::now(); - if let Some(mut prepared_block) = - prepared_candidates.lookup(&block, request.work_id(), &network) + if let Some(prepared::CachedPreparedCandidate { + source, + prepared: cached_prepared_block, + }) = prepared_candidates.lookup(&block, request.work_id(), &network) { let pow_policy = zakura_header_chain::PowPolicy::for_network(&network)?; if pow_policy.is_authenticated_custom_waiver() { @@ -326,24 +328,27 @@ where check::merkle_root_validity( &network, &block, - &prepared_block.transaction_hashes, + &cached_prepared_block.transaction_hashes, )?; metrics::histogram!("mining.solved_header_check.duration_seconds") .record(solved_header_start.elapsed().as_secs_f64()); + let mut prepared_block = cached_prepared_block.as_ref().clone(); prepared_block.block = block; prepared_block.hash = hash; prepared_block.height = height; let admission = request.admission(); - if let Some(admission) = &admission { - if check_prepared_mined_relay_eligibility( - &mut state_service, - (&prepared_block).into(), - ) - .await? - == zs::PreparedMinedRelayEligibility::Authorized - { - admission.authorize_optimistic_relay(); + if source == PreparedCandidateSource::ServerTemplate { + if let Some(admission) = &admission { + if check_prepared_mined_relay_eligibility( + &mut state_service, + (&prepared_block).into(), + ) + .await? + == zs::PreparedMinedRelayEligibility::Authorized + { + admission.authorize_optimistic_relay(); + } } } return commit_prepared_block(state_service, prepared_block, admission).await; diff --git a/crates/zakura-consensus/src/block/prepared.rs b/crates/zakura-consensus/src/block/prepared.rs index e78ce5448a..63c0d182eb 100644 --- a/crates/zakura-consensus/src/block/prepared.rs +++ b/crates/zakura-consensus/src/block/prepared.rs @@ -56,18 +56,23 @@ struct Entry { work_id: Option, fingerprint: [u8; 32], immutable_bytes: Vec, - prepared: SemanticallyVerifiedBlock, + prepared: Arc, size: usize, expires_at: Instant, } +pub(super) struct CachedPreparedCandidate { + pub source: PreparedCandidateSource, + pub prepared: Arc, +} + impl PreparedCandidateCache { pub(super) fn lookup( &self, block: &Block, work_id: Option<&str>, network: &Network, - ) -> Option { + ) -> Option { // Deriving the candidate bytes costs a full block serialization, so skip it when the // cache holds no entry that could match. { @@ -91,20 +96,24 @@ impl PreparedCandidateCache { inner.prune_expired(); if let Some(work_id) = work_id { - if let Some(entry) = inner.find_work_id(work_id) { - if entry.immutable_bytes == immutable_bytes { - metrics::counter!("mining.prepared_cache.hits").increment(1); - return Some(entry.prepared.clone()); - } - + if let Some((source, entry)) = + inner.find_work_id_candidate(work_id, fingerprint, &immutable_bytes) + { + let prepared = Arc::clone(&entry.prepared); + drop(inner); + metrics::counter!("mining.prepared_cache.hits").increment(1); + return Some(CachedPreparedCandidate { source, prepared }); + } + if inner.contains_work_id(work_id) { metrics::counter!("mining.prepared_cache.mismatches").increment(1); - return None; } } - if let Some(entry) = inner.find_candidate(fingerprint, &immutable_bytes) { + if let Some((source, entry)) = inner.find_candidate(fingerprint, &immutable_bytes) { + let prepared = Arc::clone(&entry.prepared); + drop(inner); metrics::counter!("mining.prepared_cache.hits").increment(1); - return Some(entry.prepared.clone()); + return Some(CachedPreparedCandidate { source, prepared }); } metrics::counter!("mining.prepared_cache.misses").increment(1); @@ -131,18 +140,27 @@ impl PreparedCandidateCache { if let Some(work_id) = work_id { if let Some((existing_source, entry)) = inner.find_work_id_with_source(work_id) { if entry.immutable_bytes == immutable_bytes { - return; - } - - metrics::counter!( - "mining.prepared_cache.work_id_conflicts", - "source" => source.metric_label() - ) - .increment(1); - let server_reclaims_proposal = source == PreparedCandidateSource::ServerTemplate - && existing_source == PreparedCandidateSource::ClientProposal; - if !server_reclaims_proposal { - return; + if source != PreparedCandidateSource::ServerTemplate + || existing_source == PreparedCandidateSource::ServerTemplate + { + return; + } + + inner.proposals.remove_work_id(work_id); + } else { + metrics::counter!( + "mining.prepared_cache.work_id_conflicts", + "source" => source.metric_label() + ) + .increment(1); + let server_reclaims_proposal = source + == PreparedCandidateSource::ServerTemplate + && existing_source == PreparedCandidateSource::ClientProposal; + if server_reclaims_proposal { + inner.proposals.remove_work_id(work_id); + } else if source == PreparedCandidateSource::ServerTemplate { + return; + } } } } @@ -159,12 +177,13 @@ impl PreparedCandidateCache { } else { None }; - // Count the canonical candidate, derived verification inputs, and caller-supplied work ID. - let size = immutable_bytes.len().saturating_mul(2).saturating_add( + let size = retained_size( + immutable_bytes.len(), work_id .map(str::len) .or_else(|| existing_work_id.as_ref().map(String::len)) .unwrap_or(0), + &prepared, ); let (max_entries, max_bytes) = source.limits(); if size > max_bytes { @@ -185,7 +204,13 @@ impl PreparedCandidateCache { } let partition = inner.partition(source); - partition.remove_matching(work_id.as_deref(), fingerprint, &immutable_bytes); + partition.remove_matching( + (source == PreparedCandidateSource::ServerTemplate) + .then_some(work_id.as_deref()) + .flatten(), + fingerprint, + &immutable_bytes, + ); while partition.entries.len() >= max_entries || partition.bytes.saturating_add(size) > max_bytes @@ -200,7 +225,7 @@ impl PreparedCandidateCache { work_id, fingerprint, immutable_bytes, - prepared, + prepared: Arc::new(prepared), size, expires_at: Instant::now() + ENTRY_TTL, }); @@ -226,12 +251,40 @@ impl CacheInner { } } - fn find_work_id(&self, work_id: &str) -> Option<&Entry> { + fn contains_work_id(&self, work_id: &str) -> bool { self.server .entries .iter() .chain(self.proposals.entries.iter()) - .find(|entry| entry.work_id.as_deref() == Some(work_id)) + .any(|entry| entry.work_id.as_deref() == Some(work_id)) + } + + fn find_work_id_candidate( + &self, + work_id: &str, + fingerprint: [u8; 32], + immutable_bytes: &[u8], + ) -> Option<(PreparedCandidateSource, &Entry)> { + self.server + .entries + .iter() + .find(|entry| { + entry.work_id.as_deref() == Some(work_id) + && entry.fingerprint == fingerprint + && entry.immutable_bytes == immutable_bytes + }) + .map(|entry| (PreparedCandidateSource::ServerTemplate, entry)) + .or_else(|| { + self.proposals + .entries + .iter() + .find(|entry| { + entry.work_id.as_deref() == Some(work_id) + && entry.fingerprint == fingerprint + && entry.immutable_bytes == immutable_bytes + }) + .map(|entry| (PreparedCandidateSource::ClientProposal, entry)) + }) } fn find_work_id_with_source(&self, work_id: &str) -> Option<(PreparedCandidateSource, &Entry)> { @@ -249,14 +302,27 @@ impl CacheInner { }) } - fn find_candidate(&self, fingerprint: [u8; 32], immutable_bytes: &[u8]) -> Option<&Entry> { + fn find_candidate( + &self, + fingerprint: [u8; 32], + immutable_bytes: &[u8], + ) -> Option<(PreparedCandidateSource, &Entry)> { self.server .entries .iter() - .chain(self.proposals.entries.iter()) .find(|entry| { entry.fingerprint == fingerprint && entry.immutable_bytes == immutable_bytes }) + .map(|entry| (PreparedCandidateSource::ServerTemplate, entry)) + .or_else(|| { + self.proposals + .entries + .iter() + .find(|entry| { + entry.fingerprint == fingerprint && entry.immutable_bytes == immutable_bytes + }) + .map(|entry| (PreparedCandidateSource::ClientProposal, entry)) + }) } } @@ -335,6 +401,33 @@ impl PreparedCandidateSource { } } +fn retained_size( + immutable_bytes: usize, + work_id: usize, + prepared: &SemanticallyVerifiedBlock, +) -> usize { + // Charge three serialized copies for the normalized bytes, the decoded block, and cloned + // output scripts. Add the derived map's allocated buckets and the transaction-hash array. + // This conservative cost prevents output-heavy proposals from bypassing the byte budget. + immutable_bytes + .saturating_mul(3) + .saturating_add( + prepared.new_outputs.capacity().saturating_mul( + std::mem::size_of::() + .saturating_add(std::mem::size_of::()) + .saturating_add(1), + ), + ) + .saturating_add( + prepared + .transaction_hashes + .len() + .saturating_mul(std::mem::size_of::()), + ) + .saturating_add(work_id) + .saturating_add(std::mem::size_of::()) +} + fn immutable_candidate_bytes(block: &Block, network: &Network) -> Vec { let mut header: Header = *block.header; header.time = @@ -470,7 +563,7 @@ mod tests { } #[test] - fn inserting_a_reused_work_id_does_not_replace_the_old_candidate() { + fn proposal_work_id_conflict_falls_back_to_candidate_content() { let network = Network::Mainnet; let original = test_block(); let mut replacement = original.clone(); @@ -492,18 +585,14 @@ mod tests { &network, ); - assert!(cache.lookup(&replacement, Some("work"), &network).is_none()); + assert!(cache.lookup(&replacement, Some("work"), &network).is_some()); assert!(cache.lookup(&original, Some("work"), &network).is_some()); - assert_eq!( - cache - .0 - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .server - .entries - .len(), - 1 - ); + let inner = cache + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + assert_eq!(inner.server.entries.len(), 1); + assert_eq!(inner.proposals.entries.len(), 1); } #[test] @@ -597,7 +686,7 @@ mod tests { } #[test] - fn proposal_cannot_replace_a_server_work_id() { + fn proposal_work_id_conflict_does_not_replace_server_candidate() { let network = Network::Mainnet; let cache = PreparedCandidateCache::default(); let server = distinct_block(1); @@ -618,13 +707,16 @@ mod tests { ); assert!(cache.lookup(&server, Some("shared"), &network).is_some()); - assert!(cache.lookup(&proposal, Some("shared"), &network).is_none()); + let proposal_hit = cache + .lookup(&proposal, Some("shared"), &network) + .expect("content lookup finds the conflicting proposal"); + assert_eq!(proposal_hit.source, PreparedCandidateSource::ClientProposal); let inner = cache .0 .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); assert_eq!(inner.server.entries.len(), 1); - assert!(inner.proposals.entries.is_empty()); + assert_eq!(inner.proposals.entries.len(), 1); } #[test] @@ -691,7 +783,7 @@ mod tests { } #[test] - fn identical_work_id_and_candidate_leave_the_existing_mapping_unchanged() { + fn identical_server_candidate_promotes_the_proposal_mapping() { let network = Network::Mainnet; let cache = PreparedCandidateCache::default(); let candidate = test_block(); @@ -702,14 +794,6 @@ mod tests { PreparedCandidateSource::ClientProposal, &network, ); - let original_expiration = cache - .0 - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .proposals - .entries[0] - .expires_at; - insert( &cache, &candidate, @@ -722,9 +806,8 @@ mod tests { .0 .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); - assert!(inner.server.entries.is_empty()); - assert_eq!(inner.proposals.entries.len(), 1); - assert_eq!(inner.proposals.entries[0].expires_at, original_expiration); + assert_eq!(inner.server.entries.len(), 1); + assert!(inner.proposals.entries.is_empty()); } #[test] diff --git a/crates/zakura-consensus/src/block/tests.rs b/crates/zakura-consensus/src/block/tests.rs index 78331b571a..352d7706ce 100644 --- a/crates/zakura-consensus/src/block/tests.rs +++ b/crates/zakura-consensus/src/block/tests.rs @@ -425,7 +425,7 @@ async fn proposal_validation_succeeds_when_cache_insertion_conflicts() { }) .await; assert!(commit_result.is_ok()); - assert_eq!(transaction_calls.load(Ordering::Relaxed), 3); + assert_eq!(transaction_calls.load(Ordering::Relaxed), 2); } // TODO: enable this test after implementing contextual verification diff --git a/crates/zakura-network/src/peer/connection.rs b/crates/zakura-network/src/peer/connection.rs index 5e18fc82c3..d62af91c00 100644 --- a/crates/zakura-network/src/peer/connection.rs +++ b/crates/zakura-network/src/peer/connection.rs @@ -1437,7 +1437,9 @@ where self.handle_inbound_overload(req, now, PeerError::Overloaded) .await; - } else if e.is::() { + } else if e.is::() + || e.is::() + { // # Security // // Peer requests must have a timeout. diff --git a/crates/zakura-network/src/peer_set/set.rs b/crates/zakura-network/src/peer_set/set.rs index 7888f0cf0d..f9b380022a 100644 --- a/crates/zakura-network/src/peer_set/set.rs +++ b/crates/zakura-network/src/peer_set/set.rs @@ -94,7 +94,7 @@ //! [ZIP-201]: https://zips.z.cash/zip-0201 use std::{ - collections::{HashMap, HashSet}, + collections::{HashMap, HashSet, VecDeque}, convert, fmt::Debug, marker::PhantomData, @@ -236,7 +236,7 @@ where inventory_registry: InventoryRegistry, /// Stores requests that should be routed to peers once they are ready. - queued_broadcast_all: Option<( + queued_broadcast_all: VecDeque<( Request, tokio::sync::mpsc::UnboundedSender, HashSet, @@ -389,7 +389,7 @@ where ready_services: HashMap::new(), // Request Routing inventory_registry: InventoryRegistry::new(inv_stream, config.expose_peer_addresses), - queued_broadcast_all: None, + queued_broadcast_all: VecDeque::new(), block_gossip_peer_ips: block_gossip_peer_ips.into_iter().collect(), legacy_peer_trace: LegacyPeerTrace::new(config.zakura.trace_dir.clone()), queued_sidecar_block_gossip: None, @@ -1305,12 +1305,17 @@ where async move { let results = futs.collect::>>().await; + let succeeded = results.iter().any(Result::is_ok); tracing::debug!( ok.len = results.iter().filter(|r| r.is_ok()).count(), err.len = results.iter().filter(|r| r.is_err()).count(), "sent peer request to multiple peers" ); - Ok(Response::Nil) + if results.is_empty() || succeeded { + Ok(Response::Nil) + } else { + Err(std::io::Error::other("every selected peer request failed").into()) + } } .boxed() } @@ -1332,16 +1337,23 @@ where self.send_multiple(req, selected_peers) } - /// Broadcasts the same request to all ready peers, ignoring return values. + /// Broadcasts the same request and succeeds after at least one peer accepts it. fn broadcast_all(&mut self, req: Request) -> >::Future { - let ready_peers = self.ready_services.keys().copied().collect(); + let ready_peers: Vec<_> = self.ready_services.keys().copied().collect(); + let had_ready_peers = !ready_peers.is_empty(); let send_multiple_fut = self.send_multiple(req.clone(), ready_peers); let Some(mut queued_broadcast_fut_receiver) = self.queue_broadcast_all_unready(&req) else { + if !had_ready_peers { + return async { + Err(std::io::Error::other("block broadcast had no connected peers").into()) + } + .boxed(); + } return send_multiple_fut; }; async move { - let _ = send_multiple_fut.await?; + let mut succeeded = had_ready_peers && send_multiple_fut.await.is_ok(); // Each queued item is a `send_multiple` future whose response receivers // keep the enqueued peer requests from being treated as canceled (see the // `tx.is_canceled()` skip in `Connection::handle_client_request`). @@ -1349,12 +1361,17 @@ where // connection task synchronously but holds the response receiver inside the // returned future. Dropping that future unpolled — as `.is_some() {}` did — // drops the receiver before the connection task drains the request, so it - // sees a canceled request and racily skips the block `inv`. Spawn each - // future so it is polled to completion and keeps its receivers alive. + // sees a canceled request and racily skips the block `inv`. Poll each future to + // completion so it keeps its receivers alive and contributes to the + // delivery result. while let Some(send_fut) = queued_broadcast_fut_receiver.recv().await { - tokio::spawn(send_fut); + succeeded |= send_fut.await.is_ok(); + } + if succeeded { + Ok(Response::Nil) + } else { + Err(std::io::Error::other("block broadcast reached no peers").into()) } - Ok(Response::Nil) } .boxed() } @@ -1374,9 +1391,7 @@ where let (sender, receiver) = tokio::sync::mpsc::unbounded_channel(); let unready_peers: HashSet<_> = self.cancel_handles.keys().cloned().collect(); let queued = (req.clone(), sender, unready_peers); - - // Drop the existing queued broadcast all request, if any. - self.queued_broadcast_all = Some(queued); + self.queued_broadcast_all.push_back(queued); Some(receiver) } else { @@ -1387,49 +1402,43 @@ where /// Broadcasts the same requests to all ready peers which were unready when /// [`PeerSet::broadcast_all()`] was last called, ignoring return values. fn broadcast_all_queued(&mut self) { - let Some((req, sender, mut remaining_peers)) = self.queued_broadcast_all.take() else { - return; - }; - - if sender.is_closed() { - return; - } - - remaining_peers.retain(|addr| { - !self.bans.contains(canonical_ip(addr.ip())) - && (self.ready_services.contains_key(addr) - || self.cancel_handles.contains_key(addr)) - }); - - if remaining_peers.is_empty() { - return; - } - - let peers: Vec<_> = self - .ready_services - .keys() - .filter(|ready_peer| remaining_peers.contains(ready_peer)) - .copied() - .collect(); + let mut queued = std::mem::take(&mut self.queued_broadcast_all); + while let Some((req, sender, mut remaining_peers)) = queued.pop_front() { + if sender.is_closed() { + continue; + } - if peers.is_empty() { - self.queued_broadcast_all = Some((req, sender, remaining_peers)); - return; - } + remaining_peers.retain(|addr| { + !self.bans.contains(canonical_ip(addr.ip())) + && (self.ready_services.contains_key(addr) + || self.cancel_handles.contains_key(addr)) + }); + if remaining_peers.is_empty() { + continue; + } - for peer in &peers { - remaining_peers.remove(peer); - } + let peers: Vec<_> = self + .ready_services + .keys() + .filter(|ready_peer| remaining_peers.contains(ready_peer)) + .copied() + .collect(); + for peer in &peers { + remaining_peers.remove(peer); + } - if sender - .send(self.send_multiple(req.clone(), peers).boxed()) - .is_err() - { - return; - } + if !peers.is_empty() + && sender + .send(self.send_multiple(req.clone(), peers).boxed()) + .is_err() + { + continue; + } - if !remaining_peers.is_empty() { - self.queued_broadcast_all = Some((req, sender, remaining_peers)); + if !remaining_peers.is_empty() { + self.queued_broadcast_all + .push_back((req, sender, remaining_peers)); + } } } diff --git a/crates/zakura-network/src/peer_set/set/tests/vectors.rs b/crates/zakura-network/src/peer_set/set/tests/vectors.rs index d0cbfae98f..53f70ed5bc 100644 --- a/crates/zakura-network/src/peer_set/set/tests/vectors.rs +++ b/crates/zakura-network/src/peer_set/set/tests/vectors.rs @@ -330,11 +330,13 @@ fn broadcast_all_queued_removes_banned_peers() { remaining_peers.insert(banned_addr); let (sender, mut receiver) = tokio::sync::mpsc::unbounded_channel(); - peer_set.queued_broadcast_all = Some((Request::Peers, sender, remaining_peers)); + peer_set + .queued_broadcast_all + .push_back((Request::Peers, sender, remaining_peers)); peer_set.broadcast_all_queued(); - assert!(peer_set.queued_broadcast_all.is_none()); + assert!(peer_set.queued_broadcast_all.is_empty()); assert!(matches!( receiver.try_recv(), Err(tokio::sync::mpsc::error::TryRecvError::Disconnected) @@ -375,19 +377,19 @@ fn broadcast_all_queued_removes_disconnected_peers() { let broadcast_fut = peer_set.broadcast_all(Request::AdvertiseBlockToAll(block::Hash([9; 32]))); - assert!(peer_set.queued_broadcast_all.is_some()); + assert!(!peer_set.queued_broadcast_all.is_empty()); peer_set.remove(&peer_addr); // Polling readiness processes the canceled service. Since this was the only // peer, readiness remains pending, but queue cleanup must still run. assert!(peer_set.ready().now_or_never().is_none()); - assert!(peer_set.queued_broadcast_all.is_none()); + assert!(peer_set.queued_broadcast_all.is_empty()); timeout(Duration::from_secs(1), broadcast_fut) .await .expect("broadcast should not wait for a disconnected peer") - .expect("broadcast_all should succeed"); + .expect_err("broadcast reports that no peer received the request"); }); } @@ -423,12 +425,48 @@ fn broadcast_all_queued_removes_canceled_broadcasts() { let broadcast_fut = peer_set.broadcast_all(Request::AdvertiseBlockToAll(block::Hash([11; 32]))); - assert!(peer_set.queued_broadcast_all.is_some()); + assert!(!peer_set.queued_broadcast_all.is_empty()); drop(broadcast_fut); peer_set.broadcast_all_queued(); - assert!(peer_set.queued_broadcast_all.is_none()); + assert!(peer_set.queued_broadcast_all.is_empty()); + }); +} + +#[test] +fn new_broadcast_keeps_previous_queued_delivery() { + let peer_versions = PeerVersions { + peer_versions: vec![Version::min_specified_for_upgrade( + &Network::Mainnet, + NetworkUpgrade::Nu6_2, + )], + }; + let (runtime, _init_guard) = zakura_test::init_async(); + let _guard = runtime.enter(); + let (discovered_peers, _handles) = peer_versions.mock_peer_discovery(); + let (minimum_peer_version, _best_tip_height) = + MinimumPeerVersion::with_mock_chain_tip(&Network::Mainnet); + + runtime.block_on(async move { + let (mut peer_set, _peer_set_guard) = PeerSetBuilder::new() + .with_discover(discovered_peers) + .with_minimum_peer_version(minimum_peer_version) + .build(); + let peer_addr: PeerSocketAddr = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 1).into(); + let peer_set = peer_set.ready().await.expect("peer set is always ready"); + let service = peer_set + .take_ready_service(&peer_addr) + .expect("mock peer is ready"); + peer_set.push_unready(peer_addr, service); + + let first = peer_set.broadcast_all(Request::AdvertiseBlockToAll(block::Hash([21; 32]))); + let second = peer_set.broadcast_all(Request::AdvertiseBlockToAll(block::Hash([22; 32]))); + + assert_eq!(peer_set.queued_broadcast_all.len(), 2); + drop((first, second)); + peer_set.broadcast_all_queued(); + assert!(peer_set.queued_broadcast_all.is_empty()); }); } @@ -480,7 +518,7 @@ fn broadcast_all_queued_does_not_wait_for_receiver_capacity() { peer_set.broadcast_all_queued(); } - assert!(peer_set.queued_broadcast_all.is_none()); + assert!(peer_set.queued_broadcast_all.is_empty()); let mut queued_deliveries = 0; while receiver.try_recv().is_ok() { @@ -559,53 +597,48 @@ fn mined_block_gossip_to_unready_peer_is_delivered_not_canceled() { let broadcast_handle = tokio::spawn(peer_set.broadcast_all(Request::AdvertiseBlockToAll(hash))); - // Drive the peer set so both peers re-ready and `broadcast_all_queued` - // delivers the queued gossip; yield so the spawned drain loop processes - // it. Once every queued peer has been delivered, the drain loop drains - // and the broadcast future completes — that completion is the point at - // which the delivery future has definitively been spawned (fixed) or - // dropped (buggy), so we can check the response channel deterministically. - let mut broadcast_finished = false; + // Drive the peer set until both peers receive the queued gossip. Answer each request so + // the broadcast can confirm that at least one peer accepted it. + let mut handles = [handle_1, handle_2]; + let mut delivered = 0; for _ in 0..16 { { let _ = peer_set.ready().await.expect("peer set is always ready"); } tokio::task::yield_now().await; - if broadcast_handle.is_finished() { - broadcast_finished = true; + + for handle in &mut handles { + let Some(client_request) = + handle.try_to_receive_outbound_client_request().request() + else { + continue; + }; + assert!( + matches!(client_request.request, Request::AdvertiseBlockToAll(h) if h == hash), + "expected the mined-block advertisement, got {:?}", + client_request.request, + ); + assert!( + !client_request.tx.is_canceled(), + "the queued send future must remain alive until the peer responds", + ); + client_request + .tx + .send(Ok(Response::Nil)) + .expect("the broadcast waits for the peer response"); + delivered += 1; + } + + tokio::task::yield_now().await; + if delivered == 2 && broadcast_handle.is_finished() { break; } } - assert!( - broadcast_finished, - "the mined-block broadcast future should complete once queued deliveries drain", - ); + assert_eq!(delivered, 2, "both peers receive the queued broadcast"); broadcast_handle .await .expect("broadcast task should not panic") .expect("broadcast_all should succeed"); - - // Both originally-unready peers must have received the mined-block inv, - // and — crucially — the delivery future must have been kept alive rather - // than dropped. On the buggy code the future is dropped, cancelling the - // response channel, which the connection task treats as a canceled - // request and skips the block inv. - for mut handle in [handle_1, handle_2] { - let client_request = handle - .try_to_receive_outbound_client_request() - .request() - .expect("each once-unready peer should receive the queued mined-block gossip"); - assert!( - matches!(client_request.request, Request::AdvertiseBlockToAll(h) if h == hash), - "expected the mined-block advertisement, got {:?}", - client_request.request, - ); - assert!( - !client_request.tx.is_canceled(), - "the queued send future must be spawned, not dropped: a dropped future \ - cancels the response channel and the connection skips the block inv", - ); - } }); } @@ -812,14 +845,16 @@ fn broadcast_all_queued_bans_mapped_ipv6_against_canonical_ban() { remaining_peers.insert(mapped_addr); let (sender, mut receiver) = tokio::sync::mpsc::unbounded_channel(); - peer_set.queued_broadcast_all = Some((Request::Peers, sender, remaining_peers)); + peer_set + .queued_broadcast_all + .push_back((Request::Peers, sender, remaining_peers)); peer_set.broadcast_all_queued(); // The mapped-form peer must be dropped by the (canonical) ban filter, so no peers // remain queued for the re-send and the response channel must close. On // un-canonicalized code the mapped peer would survive the filter. - assert!(peer_set.queued_broadcast_all.is_none()); + assert!(peer_set.queued_broadcast_all.is_empty()); assert!(matches!( receiver.try_recv(), Err(tokio::sync::mpsc::error::TryRecvError::Disconnected) diff --git a/crates/zakura-rpc/src/lib.rs b/crates/zakura-rpc/src/lib.rs index 67022674cd..07b39c59ff 100644 --- a/crates/zakura-rpc/src/lib.rs +++ b/crates/zakura-rpc/src/lib.rs @@ -20,5 +20,5 @@ mod tests; pub use methods::types::{ get_block_template::{fetch_chain_info, proposal::proposal_block_from_template, MinerParams}, - submit_block::{MinedBlockEvent, PendingBlockRegistry, SubmitBlockChannel}, + submit_block::{MinedBlockEvent, PendingBlockRegistry, PendingBlockSignal, SubmitBlockChannel}, }; diff --git a/crates/zakura-rpc/src/methods.rs b/crates/zakura-rpc/src/methods.rs index 3b19505501..364990d8dd 100644 --- a/crates/zakura-rpc/src/methods.rs +++ b/crates/zakura-rpc/src/methods.rs @@ -141,9 +141,6 @@ use types::{ z_validate_address::ZValidateAddressResponse, }; -/// Bounds the final mined-block event send after the RPC lifecycle has detached. -const MINED_BLOCK_EVENT_SEND_TIMEOUT: Duration = Duration::from_secs(5); - /// Calls a Tower service and maps readiness or call errors to /// [`server::error::LegacyCode::Misc`]. async fn call_service(service: S, request: Request) -> Result @@ -948,7 +945,7 @@ where latest_chain_tip: Tip, address_book: AddressBook, last_warn_error_log_rx: LoggedLastEvent, - mined_block_sender: Option>, + mined_block_sender: Option>, ) -> (Self, JoinHandle<()>) where VersionString: ToString + Clone + Send + 'static, @@ -989,7 +986,7 @@ where latest_chain_tip: Tip, address_book: AddressBook, last_warn_error_log_rx: LoggedLastEvent, - mined_block_sender: Option>, + mined_block_sender: Option>, pending_blocks: PendingBlockRegistry, ) -> (Self, JoinHandle<()>) where @@ -1047,26 +1044,35 @@ where } fn prepare_template_in_background(&self, template: &BlockTemplateResponse) { - let Some(preparation_permit) = self.gbt.try_acquire_template_preparation() else { - metrics::counter!("mining.template_preparation.saturated").increment(1); + let Some(template) = self.gbt.queue_template_preparation(template.clone()) else { + metrics::counter!("mining.template_preparation.coalesced").increment(1); return; }; - let template = template.clone(); let network = self.network.clone(); let verifier = self.gbt.block_verifier_router(); + let gbt = self.gbt.clone(); tokio::spawn( async move { - let _preparation_permit = preparation_permit; - let Ok(block) = proposal_block_from_template(&template, None, &network) else { - return; - }; - let request = zakura_consensus::Request::Prepare { - block: Arc::new(block), - work_id: Some(template.work_id().clone()), - source: zakura_consensus::PreparedCandidateSource::ServerTemplate, - }; - if let Err(error) = verifier.oneshot(request).await { - tracing::debug!(?error, "background mining candidate preparation failed"); + let mut template = template; + loop { + if let Ok(block) = proposal_block_from_template(&template, None, &network) { + let request = zakura_consensus::Request::Prepare { + block: Arc::new(block), + work_id: Some(template.work_id().clone()), + source: zakura_consensus::PreparedCandidateSource::ServerTemplate, + }; + if let Err(error) = verifier.clone().oneshot(request).await { + tracing::debug!( + ?error, + "background mining candidate preparation failed" + ); + } + } + + let Some(next) = gbt.next_template_preparation() else { + break; + }; + template = next; } } .in_current_span(), @@ -2824,8 +2830,8 @@ where tokio::pin!(verification); let admission_start = std::time::Instant::now(); - let mut early_result = None; let mut pending_registration = None; + let mut early_sent = false; let verification_result = tokio::select! { biased; @@ -2837,15 +2843,14 @@ where && optimistic_block_inventory { if let Some(registration) = pending_blocks.insert(block.clone()) { - let (advertised, receiver) = tokio::sync::oneshot::channel(); let event = MinedBlockEvent::Early { hash: block_hash, height, submitted_at, - advertised, + pending: registration.signal(), }; - if mined_block_sender.try_send(event).is_ok() { - early_result = Some(receiver); + if mined_block_sender.send(event).is_ok() { + early_sent = true; pending_registration = Some(registration); } } @@ -2864,47 +2869,14 @@ where ); } - let committed = verification_result.is_ok(); - tokio::spawn(async move { - let early_advertised = match early_result { - Some(receiver) => tokio::time::timeout(Duration::from_secs(20), receiver) - .await - .ok() - .and_then(|result| result.ok()) - .unwrap_or(false), - None => false, - }; - let event = if committed { - if optimistic_block_inventory && !early_advertised { - metrics::counter!("mining.optimistic_inventory.fallbacks").increment(1); - } - MinedBlockEvent::Committed { - hash: block_hash, - height, - early_advertised, - } - } else { - if early_advertised { - metrics::counter!("mining.optimistic_inventory.post_commit_failures") - .increment(1); - tracing::warn!( - ?block_hash, - ?height, - "mined block failed contextual commit after early inventory" - ); - } - MinedBlockEvent::Failed { + if verification_result.is_ok() { + if mined_block_sender + .send(MinedBlockEvent::Committed { hash: block_hash, height, - early_advertised, - } - }; - let send_result = tokio::time::timeout( - MINED_BLOCK_EVENT_SEND_TIMEOUT, - mined_block_sender.send(event), - ) - .await; - if !matches!(send_result, Ok(Ok(()))) { + }) + .is_err() + { metrics::counter!("mining.optimistic_inventory.final_send_failures") .increment(1); tracing::warn!( @@ -2913,7 +2885,15 @@ where "could not send the final mined-block event" ); } - }); + } else if early_sent { + metrics::counter!("mining.optimistic_inventory.post_admission_failures") + .increment(1); + tracing::warn!( + ?block_hash, + ?height, + "mined block failed contextual commit after state admission" + ); + } verification_result }); diff --git a/crates/zakura-rpc/src/methods/types/get_block_template.rs b/crates/zakura-rpc/src/methods/types/get_block_template.rs index 09de544dc1..f398631b34 100644 --- a/crates/zakura-rpc/src/methods/types/get_block_template.rs +++ b/crates/zakura-rpc/src/methods/types/get_block_template.rs @@ -10,7 +10,7 @@ mod tests; use std::{ fmt::{self}, - sync::Arc, + sync::{Arc, Mutex}, }; use derive_getters::Getters; @@ -18,7 +18,7 @@ use derive_new::new; use jsonrpsee::core::RpcResult; use jsonrpsee_types::{ErrorCode, ErrorObject}; use rand::{rngs::OsRng, RngCore}; -use tokio::sync::{mpsc, OwnedSemaphorePermit, Semaphore}; +use tokio::sync::mpsc; use tower::{Service, ServiceExt}; use zcash_keys::address::Address; use zcash_protocol::memo::MemoBytes; @@ -65,22 +65,49 @@ pub use parameters::{ }; pub use proposal::{BlockProposalResponse, BlockTemplateTimeSource}; -const MAX_BACKGROUND_TEMPLATE_PREPARATIONS: usize = 1; - #[derive(Clone, Debug)] -struct TemplatePreparationLimiter(Arc); +struct TemplatePreparationQueue(Arc>>); + +#[derive(Debug)] +struct TemplatePreparationState { + running: bool, + pending: Option, +} -impl Default for TemplatePreparationLimiter { +impl Default for TemplatePreparationQueue { fn default() -> Self { - Self(Arc::new(Semaphore::new( - MAX_BACKGROUND_TEMPLATE_PREPARATIONS, - ))) + Self(Arc::new(Mutex::new(TemplatePreparationState { + running: false, + pending: None, + }))) } } -impl TemplatePreparationLimiter { - fn try_acquire(&self) -> Option { - self.0.clone().try_acquire_owned().ok() +impl TemplatePreparationQueue { + fn enqueue(&self, template: T) -> Option { + let mut state = self + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if state.running { + state.pending = Some(template); + None + } else { + state.running = true; + Some(template) + } + } + + fn next_or_finish(&self) -> Option { + let mut state = self + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let next = state.pending.take(); + if next.is_none() { + state.running = false; + } + next } } @@ -589,7 +616,7 @@ where /// A channel to send successful block submissions to the block gossip task, /// so they can be advertised to peers. - mined_block_sender: mpsc::Sender, + mined_block_sender: mpsc::UnboundedSender, /// Blocks whose hashes were advertised before contextual commit completed. pending_blocks: PendingBlockRegistry, @@ -597,8 +624,8 @@ where /// Whether state admission can trigger an early inventory. optimistic_block_inventory: bool, - /// Limits detached template preparation work. - template_preparation_limiter: TemplatePreparationLimiter, + /// Coalesces detached template preparation work to the newest template. + template_preparation_queue: TemplatePreparationQueue, } impl GetBlockTemplateHandler @@ -612,7 +639,7 @@ where conf: config::mining::Config, block_verifier_router: BlockVerifierRouter, sync_status: SyncStatus, - mined_block_sender: Option>, + mined_block_sender: Option>, pending_blocks: PendingBlockRegistry, ) -> Self { let optimistic_block_inventory = conf.optimistic_block_inventory; @@ -624,7 +651,7 @@ where .unwrap_or(SubmitBlockChannel::default().sender()), pending_blocks, optimistic_block_inventory, - template_preparation_limiter: TemplatePreparationLimiter::default(), + template_preparation_queue: TemplatePreparationQueue::default(), } } @@ -644,7 +671,7 @@ where } /// Returns a sender for the owned mined-block lifecycle task. - pub fn mined_block_sender(&self) -> mpsc::Sender { + pub fn mined_block_sender(&self) -> mpsc::UnboundedSender { self.mined_block_sender.clone() } @@ -658,9 +685,17 @@ where self.optimistic_block_inventory } - /// Reserves the background template preparation slot. - pub(crate) fn try_acquire_template_preparation(&self) -> Option { - self.template_preparation_limiter.try_acquire() + /// Queues a server template and returns the first item for a new worker. + pub(crate) fn queue_template_preparation( + &self, + template: BlockTemplateResponse, + ) -> Option { + self.template_preparation_queue.enqueue(template) + } + + /// Returns the newest queued template or marks the worker idle. + pub(crate) fn next_template_preparation(&self) -> Option { + self.template_preparation_queue.next_or_finish() } /// Randomizes the coinbase data, if miner parameters are set. diff --git a/crates/zakura-rpc/src/methods/types/get_block_template/tests.rs b/crates/zakura-rpc/src/methods/types/get_block_template/tests.rs index 3640eef12b..339eb3d570 100644 --- a/crates/zakura-rpc/src/methods/types/get_block_template/tests.rs +++ b/crates/zakura-rpc/src/methods/types/get_block_template/tests.rs @@ -24,19 +24,17 @@ use zakura_chain::{ use crate::client::TransactionTemplate; use crate::config::mining::{default_miner_address, MinerAddressType}; -use super::{MinerParams, TemplatePreparationLimiter}; +use super::{MinerParams, TemplatePreparationQueue}; #[test] -fn template_preparation_is_single_flight() { - let limiter = TemplatePreparationLimiter::default(); - let permit = limiter - .try_acquire() - .expect("the first preparation reserves the slot"); - - assert!(limiter.try_acquire().is_none()); - - drop(permit); - assert!(limiter.try_acquire().is_some()); +fn template_preparation_queue_keeps_the_latest_pending_template() { + let queue = TemplatePreparationQueue::::default(); + + assert_eq!(queue.enqueue(1), Some(1)); + assert_eq!(queue.enqueue(2), None); + assert_eq!(queue.enqueue(3), None); + assert_eq!(queue.next_or_finish(), Some(3)); + assert_eq!(queue.next_or_finish(), None); } /// Tests transparent coinbase generation at every configured Sapling-and-later diff --git a/crates/zakura-rpc/src/methods/types/submit_block.rs b/crates/zakura-rpc/src/methods/types/submit_block.rs index 7d1635c890..16f9465d0f 100644 --- a/crates/zakura-rpc/src/methods/types/submit_block.rs +++ b/crates/zakura-rpc/src/methods/types/submit_block.rs @@ -9,7 +9,7 @@ use std::{ time::Duration, }; -use tokio::sync::{mpsc, oneshot, watch, OwnedSemaphorePermit, Semaphore}; +use tokio::sync::{mpsc, watch}; use zakura_chain::block; @@ -42,7 +42,6 @@ pub struct SubmitBlockParameters { pub const PENDING_BLOCK_WAIT: Duration = Duration::from_secs(15); const MAX_PENDING_BLOCKS: usize = 16; -const MAX_PENDING_BLOCK_WAITS: usize = 32; /// A mined-block lifecycle event consumed by the block gossip task. #[derive(Debug)] @@ -55,8 +54,8 @@ pub enum MinedBlockEvent { height: block::Height, /// When the RPC accepted the submitted bytes. submitted_at: std::time::Instant, - /// Reports whether the early network advertisement completed. - advertised: oneshot::Sender, + /// Cancels the advertisement if contextual verification rejects the block. + pending: PendingBlockSignal, }, /// The contextual commit completed. Committed { @@ -64,17 +63,6 @@ pub enum MinedBlockEvent { hash: block::Hash, /// The block height. height: block::Height, - /// Whether the early advertisement completed successfully. - early_advertised: bool, - }, - /// The contextual commit failed after state admission. - Failed { - /// The block hash. - hash: block::Hash, - /// The block height. - height: block::Height, - /// Whether peers received an early inventory. - early_advertised: bool, }, } @@ -91,11 +79,10 @@ struct PendingBlock { status: watch::Sender, } -/// Stores pending blocks and bounds peer waits. +/// Stores pending blocks and coalesces peer waits by block hash. #[derive(Debug)] struct PendingBlockRegistryInner { entries: Mutex>, - wait_permits: Arc, next_owner_id: AtomicU64, } @@ -107,12 +94,38 @@ impl Default for PendingBlockRegistry { fn default() -> Self { Self(Arc::new(PendingBlockRegistryInner { entries: Mutex::new(HashMap::new()), - wait_permits: Arc::new(Semaphore::new(MAX_PENDING_BLOCK_WAITS)), next_owner_id: AtomicU64::new(1), })) } } +/// Reports whether an early-advertised block remains valid. +#[derive(Debug)] +pub struct PendingBlockSignal(watch::Receiver); + +impl PendingBlockSignal { + /// Returns true unless contextual verification has rejected the block. + pub fn is_valid(&self) -> bool { + !matches!(*self.0.borrow(), PendingStatus::Failed) + } + + /// Resolves when contextual verification rejects the block. + pub async fn wait_for_failure(&mut self) { + loop { + let status = self.0.borrow_and_update().clone(); + match status { + PendingStatus::Failed => return, + PendingStatus::Committed(_) => std::future::pending::<()>().await, + PendingStatus::Waiting => {} + } + + if self.0.changed().await.is_err() { + std::future::pending::<()>().await; + } + } + } +} + /// Owns one pending-block registry entry. #[derive(Debug)] pub(crate) struct PendingBlockRegistration { @@ -123,6 +136,23 @@ pub(crate) struct PendingBlockRegistration { } impl PendingBlockRegistration { + /// Returns a signal that cancels stale early inventory after commit failure. + pub(crate) fn signal(&self) -> PendingBlockSignal { + let entries = self + .registry + .0 + .entries + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let status = entries + .get(&self.hash) + .filter(|entry| entry.owner_id == self.owner_id) + .expect("registration owns its entry until it resolves") + .status + .subscribe(); + PendingBlockSignal(status) + } + /// Resolves this registration and wakes its peer waiters. pub(crate) fn resolve(mut self, result: Result, ()>) { self.registry.resolve(self.hash, self.owner_id, result); @@ -174,9 +204,9 @@ impl PendingBlockRegistry { .entries .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); - if !entries + if entries .get(&hash) - .is_some_and(|entry| entry.owner_id == owner_id) + .is_none_or(|entry| entry.owner_id != owner_id) { return; } @@ -204,17 +234,10 @@ impl PendingBlockRegistry { .unwrap_or_else(std::sync::PoisonError::into_inner) .get(&hash) .map(|entry| entry.status.subscribe()); - let wait_permits = status.as_ref().map(|_| self.0.wait_permits.clone()); let deadline = tokio::time::Instant::now() + PENDING_BLOCK_WAIT; async move { let mut status = status?; - let _wait_permit: OwnedSemaphorePermit = wait_permits? - .try_acquire_owned() - .map_err(|_| { - metrics::counter!("mining.pending_peer_wait.saturated").increment(1); - }) - .ok()?; let start = std::time::Instant::now(); let result = tokio::time::timeout_at(deadline, async { loop { @@ -283,33 +306,28 @@ impl From for SubmitBlockResponse { /// A submit block channel, used to inform the gossip task about mined blocks. pub struct SubmitBlockChannel { /// The channel sender - sender: mpsc::Sender, + sender: mpsc::UnboundedSender, /// The channel receiver - receiver: mpsc::Receiver, + receiver: mpsc::UnboundedReceiver, } impl SubmitBlockChannel { /// Creates a new submit block channel pub fn new() -> Self { - /// How many unread messages the submit block channel should buffer before rejecting sends. - /// - /// This should be large enough to usually avoid rejecting sends. This channel is used by - /// the block hash gossip task, which waits for a ready peer in the peer set while - /// processing messages from this channel and could be much slower to gossip block hashes - /// than it is to commit blocks and produce new block templates. - const SUBMIT_BLOCK_CHANNEL_CAPACITY: usize = 10_000; - - let (sender, receiver) = mpsc::channel(SUBMIT_BLOCK_CHANNEL_CAPACITY); + // Only admitted early events and successful commit events enter this channel. Invalid and + // duplicate submissions cannot fill it, and the gossip task does not wait for peer + // readiness while consuming it. + let (sender, receiver) = mpsc::unbounded_channel(); Self { sender, receiver } } /// Get the channel sender - pub fn sender(&self) -> mpsc::Sender { + pub fn sender(&self) -> mpsc::UnboundedSender { self.sender.clone() } /// Get the channel receiver - pub fn receiver(self) -> mpsc::Receiver { + pub fn receiver(self) -> mpsc::UnboundedReceiver { self.receiver } } @@ -386,7 +404,22 @@ mod tests { } #[tokio::test] - async fn pending_block_waits_are_bounded() { + async fn pending_block_failure_cancels_stale_inventory() { + let registry = PendingBlockRegistry::default(); + let block = test_block(); + let registration = registry + .insert(block) + .expect("the registry accepts the block"); + let mut signal = registration.signal(); + + assert!(signal.is_valid()); + registration.resolve(Err(())); + signal.wait_for_failure().await; + assert!(!signal.is_valid()); + } + + #[tokio::test] + async fn pending_block_waits_for_one_hash_are_coalesced() { let registry = PendingBlockRegistry::default(); let block = test_block(); let hash = block.hash(); @@ -394,24 +427,11 @@ mod tests { .insert(block.clone()) .expect("the registry accepts the block"); - let waits: Vec<_> = (0..MAX_PENDING_BLOCK_WAITS) - .map(|_| { - let registry = registry.clone(); - tokio::spawn(async move { registry.wait(hash).await }) - }) - .collect(); - while registry.0.wait_permits.available_permits() > 0 { - tokio::task::yield_now().await; - } - assert_eq!(registry.wait(hash).await, None); - - for wait in waits { - wait.abort(); - let _ = wait.await; - } - let wait = registry.wait(hash); + let waits: Vec<_> = (0..64).map(|_| registry.wait(hash)).collect(); registration.resolve(Ok(block.clone())); - assert_eq!(wait.await, Some(block)); + for result in futures::future::join_all(waits).await { + assert_eq!(result, Some(block.clone())); + } } #[test] diff --git a/crates/zakura-state/src/error.rs b/crates/zakura-state/src/error.rs index d598a82229..d7da8b4658 100644 --- a/crates/zakura-state/src/error.rs +++ b/crates/zakura-state/src/error.rs @@ -275,6 +275,10 @@ pub enum CommitBlockError { error: String, }, + /// The orphan queue reached its memory bound. + #[error("too many blocks are waiting for unavailable parents")] + QueueFull, + /// The write task exited (likely during shutdown). #[error("block commit task exited. Is Zakura shutting down?")] #[non_exhaustive] @@ -339,6 +343,9 @@ impl CommitBlockError { Self::HeaderChainError { .. } => { BodyVerificationClass::Retryable(TransientBodyFailureKind::Storage) } + Self::QueueFull => { + BodyVerificationClass::Retryable(TransientBodyFailureKind::VerifierUnavailable) + } Self::WriteTaskExited => { BodyVerificationClass::Retryable(TransientBodyFailureKind::VerifierUnavailable) } diff --git a/crates/zakura-state/src/request.rs b/crates/zakura-state/src/request.rs index ad98287d95..03de014972 100644 --- a/crates/zakura-state/src/request.rs +++ b/crates/zakura-state/src/request.rs @@ -80,7 +80,12 @@ impl BlockAdmission { } /// Marks the block as admitted to the active non-finalized write queue. - pub(crate) fn admit(&self) { + pub(crate) fn admit(&self, optimistic_relay_still_authorized: bool) { + if !optimistic_relay_still_authorized { + self.0 + .optimistic_relay_authorized + .store(false, Ordering::Release); + } if self .0 .state @@ -124,10 +129,12 @@ impl BlockAdmission { pub async fn wait(&self) -> bool { loop { let notified = self.0.changed.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); match self.0.state.load(Ordering::Acquire) { Self::ADMITTED => return true, Self::REJECTED => return false, - Self::PENDING => notified.await, + Self::PENDING => notified.as_mut().await, _ => unreachable!("block admission state only uses declared constants"), } } @@ -847,13 +854,19 @@ mod tests { rejected.authorize_optimistic_relay(); assert!(rejected.optimistic_relay_authorized()); rejected.reject(); - rejected.admit(); + rejected.admit(true); assert!(!rejected.wait().await); let admitted = BlockAdmission::pending(); - admitted.admit(); + admitted.admit(true); admitted.reject(); assert!(admitted.wait().await); + + let stale = BlockAdmission::pending(); + stale.authorize_optimistic_relay(); + stale.admit(false); + assert!(stale.wait().await); + assert!(!stale.optimistic_relay_authorized()); } } diff --git a/crates/zakura-state/src/service.rs b/crates/zakura-state/src/service.rs index 9cb61c1fba..48847ef241 100644 --- a/crates/zakura-state/src/service.rs +++ b/crates/zakura-state/src/service.rs @@ -15,7 +15,7 @@ //! chain tip changes. use std::{ - collections::{BTreeMap, HashMap}, + collections::{BTreeMap, HashMap, HashSet}, future::Future, ops::Bound, path::PathBuf, @@ -179,6 +179,9 @@ pub(crate) struct StateService { /// Hashes of blocks below the finalized tip height are periodically pruned. non_finalized_block_write_sent_hashes: SentHashes, + /// Parents targeted by operator invalidation cannot authorize optimistic relay. + optimistic_relay_blocked_parents: HashSet, + /// If an invalid block is sent on `finalized_block_write_sender` /// or `non_finalized_block_write_sender`, /// this channel gets the [`block::Hash`] of the valid tip. @@ -562,6 +565,7 @@ impl StateService { block_write_sender, finalized_block_write_last_sent_hash, non_finalized_block_write_sent_hashes, + optimistic_relay_blocked_parents: HashSet::new(), invalid_block_write_reset_receiver, non_finalized_rejected_receiver, pending_utxos, @@ -919,10 +923,11 @@ impl StateService { fn queue_and_commit_to_non_finalized_state( &mut self, semantically_verified: SemanticallyVerifiedBlock, - mut admission: Option, + admission: Option, ) -> oneshot::Receiver> { tracing::debug!(block = %semantically_verified.block, "queueing block for contextual verification"); let parent_hash = semantically_verified.block.header.previous_block_hash; + let hash = semantically_verified.hash; // Drop hashes of any blocks the write task has rejected before checking // the SentHashes membership below. Without this, a rejected same-hash @@ -966,23 +971,33 @@ impl StateService { // [`Request::CommitSemanticallyVerifiedBlock`] contract: a request to commit a block which // has been queued but not yet committed to the state fails the older request and replaces // it with the newer request. - let rsp_rx = if let Some((_, old_rsp_tx, old_admission)) = self + let rsp_rx = if self .non_finalized_state_queued_blocks .get_mut(&semantically_verified.hash) + .is_some() { tracing::debug!("replacing older queued request with new request"); - let (mut rsp_tx, rsp_rx) = oneshot::channel(); - std::mem::swap(old_rsp_tx, &mut rsp_tx); - std::mem::swap(old_admission, &mut admission); - if let Some(admission) = admission { - admission.reject(); + let (rsp_tx, rsp_rx) = oneshot::channel(); + let (_, old_rsp_tx, old_admission) = self.non_finalized_state_queued_blocks.replace( + semantically_verified.hash, + (semantically_verified, rsp_tx, admission), + ); + if let Some(old_admission) = old_admission { + old_admission.reject(); } - let _ = rsp_tx.send(Err(CommitBlockError::new_duplicate( - Some(semantically_verified.hash.into()), + let _ = old_rsp_tx.send(Err(CommitBlockError::new_duplicate( + Some(hash.into()), KnownBlock::Queue, ) .into())); rsp_rx + } else if self.non_finalized_state_queued_blocks.is_full() { + if let Some(admission) = admission { + admission.reject(); + } + let (rsp_tx, rsp_rx) = oneshot::channel(); + let _ = rsp_tx.send(Err(CommitBlockError::QueueFull.into())); + rsp_rx } else { let (rsp_tx, rsp_rx) = oneshot::channel(); self.non_finalized_state_queued_blocks.queue(( @@ -1058,6 +1073,26 @@ impl StateService { self.non_finalized_block_write_sent_hashes .add(&queued_child.0); let admission = queued_child.2.clone(); + let candidate_parent = queued_child.0.block.header.previous_block_hash; + let optimistic_relay_still_authorized = admission + .as_ref() + .is_some_and(BlockAdmission::optimistic_relay_authorized) + && self + .best_tip() + .is_some_and(|(_, tip_hash)| tip_hash == candidate_parent) + && (self + .non_finalized_block_write_sent_hashes + .contains(&candidate_parent) + || self.read_service.db.finalized_tip_hash() == candidate_parent) + && !self + .optimistic_relay_blocked_parents + .contains(&candidate_parent); + if optimistic_relay_still_authorized { + // Only the first server candidate can reserve early relay for this parent. + // Siblings receive the normal committed relay after contextual validation. + self.optimistic_relay_blocked_parents + .insert(candidate_parent); + } let send_result = non_finalized_block_write_sender.send(queued_child.into()); if let Err(SendError(NonFinalizedWriteMessage::Commit(queued))) = send_result { @@ -1073,7 +1108,7 @@ impl StateService { }; if let Some(admission) = admission { - admission.admit(); + admission.admit(optimistic_relay_still_authorized); } new_parents.push(hash); @@ -1090,7 +1125,7 @@ impl StateService { } fn send_invalidate_block( - &self, + &mut self, hash: block::Hash, ) -> oneshot::Receiver> { let (rsp_tx, rsp_rx) = oneshot::channel(); @@ -1100,6 +1135,10 @@ impl StateService { return rsp_rx; }; + // Block optimistic relay before the writer processes the invalidation. The write channel + // preserves request order, so a later candidate cannot advertise using this stale parent. + self.optimistic_relay_blocked_parents.insert(hash); + if let Err(tokio::sync::mpsc::error::SendError(error)) = sender.send(NonFinalizedWriteMessage::Invalidate { hash, rsp_tx }) { diff --git a/crates/zakura-state/src/service/queued_blocks.rs b/crates/zakura-state/src/service/queued_blocks.rs index 749913cb85..641c8d4aa4 100644 --- a/crates/zakura-state/src/service/queued_blocks.rs +++ b/crates/zakura-state/src/service/queued_blocks.rs @@ -19,6 +19,9 @@ use crate::{ #[cfg(test)] mod tests; +/// Bounds semantically verified blocks retained while their parents are unavailable. +pub(crate) const MAX_QUEUED_BLOCKS: usize = crate::MAX_BLOCK_REORG_HEIGHT as usize; + /// A queued checkpoint verified block, and its corresponding [`Result`] channel. pub type QueuedCheckpointVerified = ( CheckpointVerifiedBlock, @@ -46,6 +49,11 @@ pub struct QueuedBlocks { } impl QueuedBlocks { + /// Returns true when the orphan queue cannot retain another distinct block. + pub fn is_full(&self) -> bool { + self.blocks.len() >= MAX_QUEUED_BLOCKS + } + /// Queue a block for eventual verification and commit. /// /// # Panics @@ -201,6 +209,36 @@ impl QueuedBlocks { self.blocks.get_mut(hash) } + /// Replaces a same-hash queued request and its retained body. + pub fn replace( + &mut self, + hash: block::Hash, + new: QueuedSemanticallyVerified, + ) -> QueuedSemanticallyVerified { + let old = self + .blocks + .insert(hash, new) + .expect("replacement hash exists in the queue"); + let replacement = self + .blocks + .get(&hash) + .expect("replacement was inserted under the same hash"); + assert_eq!(old.0.height, replacement.0.height); + assert_eq!( + old.0.block.header.previous_block_hash, + replacement.0.block.header.previous_block_hash + ); + + for outpoint in old.0.new_outputs.keys() { + self.known_utxos.remove(outpoint); + } + for (outpoint, ordered_utxo) in &replacement.0.new_outputs { + self.known_utxos + .insert(*outpoint, ordered_utxo.utxo.clone()); + } + old + } + /// Update metrics after the queue is modified fn update_metrics(&self) { if let Some(min_height) = self.by_height.keys().next() { diff --git a/crates/zakura-state/src/service/queued_blocks/tests/vectors.rs b/crates/zakura-state/src/service/queued_blocks/tests/vectors.rs index bff3b1ae0a..60ac2549b4 100644 --- a/crates/zakura-state/src/service/queued_blocks/tests/vectors.rs +++ b/crates/zakura-state/src/service/queued_blocks/tests/vectors.rs @@ -9,7 +9,9 @@ use zakura_test::prelude::*; use crate::{ arbitrary::Prepare, - service::queued_blocks::{QueuedBlocks, QueuedSemanticallyVerified, SentHashes}, + service::queued_blocks::{ + QueuedBlocks, QueuedSemanticallyVerified, SentHashes, MAX_QUEUED_BLOCKS, + }, tests::FakeChainHelper, }; @@ -96,6 +98,44 @@ fn dequeue_gives_right_children() -> Result<()> { Ok(()) } +#[test] +fn same_hash_replacement_keeps_the_new_body() -> Result<()> { + let block: Arc = + zakura_test::vectors::BLOCK_MAINNET_419200_BYTES.zcash_deserialize_into()?; + let replacement_block = Arc::new((*block).clone()); + let mut queue = QueuedBlocks::default(); + queue.queue(block.clone().into_queued()); + + let old = queue.replace(block.hash(), replacement_block.clone().into_queued()); + assert!(Arc::ptr_eq(&old.0.block, &block)); + assert!(Arc::ptr_eq( + &queue + .get_mut(&block.hash()) + .expect("replacement remains queued") + .0 + .block, + &replacement_block + )); + Ok(()) +} + +#[test] +fn orphan_queue_has_a_fixed_entry_bound() -> Result<()> { + let block: Arc = + zakura_test::vectors::BLOCK_MAINNET_419200_BYTES.zcash_deserialize_into()?; + let mut queue = QueuedBlocks::default(); + assert!(!queue.is_full()); + + for index in 0..MAX_QUEUED_BLOCKS { + let mut queued = block.clone().into_queued(); + let index = u64::try_from(index).expect("the queue bound fits in u64"); + queued.0.hash.0[..8].copy_from_slice(&index.to_le_bytes()); + queue.blocks.insert(queued.0.hash, queued); + } + assert!(queue.is_full()); + Ok(()) +} + #[test] fn prune_removes_right_children() -> Result<()> { let _init_guard = zakura_test::init(); diff --git a/crates/zakurad/src/components/inbound/tests/real_peer_set.rs b/crates/zakurad/src/components/inbound/tests/real_peer_set.rs index 96038dfae7..ab326c41dc 100644 --- a/crates/zakurad/src/components/inbound/tests/real_peer_set.rs +++ b/crates/zakurad/src/components/inbound/tests/real_peer_set.rs @@ -1203,9 +1203,7 @@ mod submitblock_test { .send(MinedBlockEvent::Committed { hash: block::Hash([1; 32]), height: block::Height(1), - early_advertised: false, }) - .await .unwrap(); let gossip_task_handle = tokio::spawn( sync::gossip_best_tip_block_hashes( diff --git a/crates/zakurad/src/components/sync/gossip.rs b/crates/zakurad/src/components/sync/gossip.rs index 65ab336141..b4dab249d2 100644 --- a/crates/zakurad/src/components/sync/gossip.rs +++ b/crates/zakurad/src/components/sync/gossip.rs @@ -7,7 +7,7 @@ use std::{future::Future, time::Duration}; use futures::TryFutureExt; use thiserror::Error; use tokio::sync::{mpsc, watch}; -use tower::{timeout::Timeout, Service, ServiceExt}; +use tower::{Service, ServiceExt}; use tracing::Instrument; use zakura_chain::block; @@ -22,14 +22,6 @@ use crate::{ use BlockGossipError::*; -/// How many completed mined block broadcasts can wait to mark the chain tip. -/// In normal operations, we expect at most 1 pending mark. -/// The main loop can be busy for several seconds in the committed-tip path. -/// During that window, multiple mined-block broadcasts could finish and -/// try to send marks. A capacity of 16 with 25-75s block times -/// is chosen arbitrarily high to be safe. -const MINED_BLOCK_MARK_CHANNEL_CAPACITY: usize = 16; - #[derive(Debug)] enum GossipEvent { MinedBlockBroadcastCompleted(block::Hash), @@ -38,8 +30,8 @@ enum GossipEvent { } async fn next_gossip_event( - mined_block_receiver: Option<&mut mpsc::Receiver>, - mined_block_mark_receiver: &mut mpsc::Receiver, + mined_block_receiver: Option<&mut mpsc::UnboundedReceiver>, + mined_block_mark_receiver: &mut mpsc::UnboundedReceiver, committed_tip_fut: impl Future, ) -> GossipEvent { if let Some(mined_block_receiver) = mined_block_receiver { @@ -81,9 +73,6 @@ pub enum BlockGossipError { #[error("sync status sender was dropped")] SyncStatus(watch::error::RecvError), - - #[error("permanent peer set failure")] - PeerSetReadiness(zn::BoxError), } /// Run continuously, gossiping newly verified [`block::Hash`]es to peers. @@ -103,7 +92,7 @@ pub async fn gossip_best_tip_block_hashes( sync_status: SyncStatus, mut chain_state: ChainTipChange, broadcast_network: ZN, - mut mined_block_receiver: Option>, + mut mined_block_receiver: Option>, ) -> Result<(), BlockGossipError> where ZN: Service + Send + Clone + 'static, @@ -111,12 +100,7 @@ where { info!("initializing block gossip task"); - // use the same timeout as tips requests, - // so broadcasts don't delay the syncer too long - let mut broadcast_network = Timeout::new(broadcast_network, TIPS_RESPONSE_TIMEOUT); - - let (mined_block_mark_sender, mut mined_block_mark_receiver) = - mpsc::channel(MINED_BLOCK_MARK_CHANNEL_CAPACITY); + let (mined_block_mark_sender, mut mined_block_mark_receiver) = mpsc::unbounded_channel(); loop { // Drain local completion notifications from spawned mined-block @@ -178,7 +162,7 @@ where // Prefer mined-block completions and submissions when multiple // branches are ready. The committed-tip path is a fallback, so // selecting it first can duplicate a mined-block broadcast. - let (((hash, height), log_msg, updated_chain_state), is_block_submission, early_ack) = + let (((hash, height), log_msg, updated_chain_state), is_block_submission, early) = match next_gossip_event( mined_block_receiver.as_mut(), &mut mined_block_mark_receiver, @@ -194,7 +178,7 @@ where hash, height, submitted_at, - advertised, + pending, }) => ( ( (hash, height), @@ -202,21 +186,9 @@ where chain_state, ), true, - Some((advertised, submitted_at)), + Some((pending, submitted_at)), ), - GossipEvent::MinedBlock(MinedBlockEvent::Committed { - hash, - height: _, - early_advertised: true, - }) => { - chain_state.mark_last_change_hash(hash); - continue; - } - GossipEvent::MinedBlock(MinedBlockEvent::Committed { - hash, - height, - early_advertised: false, - }) => ( + GossipEvent::MinedBlock(MinedBlockEvent::Committed { hash, height }) => ( ( (hash, height), "sending committed mined block broadcast", @@ -225,19 +197,6 @@ where true, None, ), - GossipEvent::MinedBlock(MinedBlockEvent::Failed { - hash, - height, - early_advertised, - }) => { - tracing::debug!( - ?hash, - ?height, - early_advertised, - "mined block lifecycle failed" - ); - continue; - } GossipEvent::CommittedTip(tip_change_close_to_network_tip) => { (tip_change_close_to_network_tip?, false, None) } @@ -256,36 +215,42 @@ where }; info!(?height, ?request, log_msg); - let broadcast_fut = broadcast_network - .ready() - .await - .map_err(PeerSetReadiness)? - .call(request); - - // Await the broadcast future in a spawned task to avoid waiting on - // `AdvertiseBlockToAll` requests when there are unready peers. - // Broadcast requests don't return errors, and we'd just want to ignore them anyway. - if is_block_submission { - let mark_tx = mined_block_mark_sender.clone(); - let submission_hash = hash; - tokio::spawn(async move { - let succeeded = broadcast_fut.await.is_ok(); - if succeeded { - let _ = mark_tx.send(submission_hash).await; - } - if let Some((advertised, submitted_at)) = early_ack { - if succeeded { - metrics::counter!("mining.optimistic_inventory.early_inventories") - .increment(1); - metrics::histogram!("mining.submit_to_inventory.duration_seconds") - .record(submitted_at.elapsed().as_secs_f64()); + // Include readiness in the deadline. The event loop must keep consuming lifecycle and tip + // events when the peer set has no ready service. + let network = broadcast_network.clone(); + let mark_tx = mined_block_mark_sender.clone(); + tokio::spawn(async move { + let broadcast = async move { + tokio::time::timeout(TIPS_RESPONSE_TIMEOUT, network.oneshot(request)) + .await + .is_ok_and(|result| result.is_ok()) + }; + let succeeded = match early { + Some((mut pending, submitted_at)) => { + if !pending.is_valid() { + false + } else { + let succeeded = tokio::select! { + biased; + _ = pending.wait_for_failure() => false, + succeeded = broadcast => succeeded, + }; + if succeeded { + metrics::counter!("mining.optimistic_inventory.early_inventories") + .increment(1); + metrics::histogram!("mining.submit_to_inventory.duration_seconds") + .record(submitted_at.elapsed().as_secs_f64()); + } + succeeded } - let _ = advertised.send(succeeded); } - }); - } else { - tokio::spawn(broadcast_fut); - } + None => broadcast.await, + }; + + if succeeded && is_block_submission { + let _ = mark_tx.send(hash); + } + }); } } @@ -308,18 +273,16 @@ mod tests { let submitted_hash = block::Hash([1; 32]); for _ in 0..READY_EVENT_ATTEMPTS { - let (mined_block_sender, mut mined_block_receiver) = mpsc::channel(1); - let (mark_sender, mut mark_receiver) = mpsc::channel(1); + let (mined_block_sender, mut mined_block_receiver) = mpsc::unbounded_channel(); + let (mark_sender, mut mark_receiver) = mpsc::unbounded_channel(); mined_block_sender .send(MinedBlockEvent::Committed { hash: submitted_hash, height: block::Height(1), - early_advertised: false, }) - .await .unwrap(); - mark_sender.send(submitted_hash).await.unwrap(); + mark_sender.send(submitted_hash).unwrap(); let event = next_gossip_event( Some(&mut mined_block_receiver), @@ -345,7 +308,6 @@ mod tests { GossipEvent::MinedBlock(MinedBlockEvent::Committed { hash, height: block::Height(1), - early_advertised: false, }) if hash == submitted_hash )); diff --git a/crates/zakurad/src/components/sync/tests/gossip.rs b/crates/zakurad/src/components/sync/tests/gossip.rs index de55aef214..507eb88f0b 100644 --- a/crates/zakurad/src/components/sync/tests/gossip.rs +++ b/crates/zakurad/src/components/sync/tests/gossip.rs @@ -27,7 +27,7 @@ const MAX_PEER_SET_REQUEST_DELAY: Duration = Duration::from_secs(30); struct GossipTestSetup { peer_set: MockService, - submitblock_sender: tokio::sync::mpsc::Sender, + submitblock_sender: tokio::sync::mpsc::UnboundedSender, state_service: BoxService, gossip_task_handle: JoinHandle>, } @@ -144,9 +144,7 @@ async fn mined_block_marks_tip_after_successful_broadcast() { .send(MinedBlockEvent::Committed { hash: block_two.hash(), height: block_two.coinbase_height().unwrap(), - early_advertised: false, }) - .await .expect("mined block notification should be accepted"); peer_set @@ -191,12 +189,7 @@ async fn mined_block_mark_survives_pending_submit_queue() { // First mined notification — start AdvertiseBlockToAll but hold the response open. submitblock_sender - .send(MinedBlockEvent::Committed { - hash, - height, - early_advertised: false, - }) - .await + .send(MinedBlockEvent::Committed { hash, height }) .expect("mined block notification should be accepted"); let first_broadcast = peer_set @@ -206,12 +199,7 @@ async fn mined_block_mark_survives_pending_submit_queue() { // Queue a second notification while the first broadcast is still in flight so the // submit-block channel is nonempty when the first mark arrives. submitblock_sender - .send(MinedBlockEvent::Committed { - hash, - height, - early_advertised: false, - }) - .await + .send(MinedBlockEvent::Committed { hash, height }) .expect("second mined block notification should be accepted"); first_broadcast.respond(Response::Nil); @@ -260,9 +248,7 @@ async fn mined_block_broadcast_timeout_uses_committed_tip_fallback() { .send(MinedBlockEvent::Committed { hash: block_two.hash(), height: block_two.coinbase_height().unwrap(), - early_advertised: false, }) - .await .expect("mined block notification should be accepted"); let slow_broadcast = peer_set diff --git a/crates/zakurad/tests/acceptance.rs b/crates/zakurad/tests/acceptance.rs index 7bdf9dac0d..f2986dea80 100644 --- a/crates/zakurad/tests/acceptance.rs +++ b/crates/zakurad/tests/acceptance.rs @@ -3824,14 +3824,7 @@ async fn nu6_funding_streams_and_coinbase_balance() -> Result<()> { let submit_block_channel_data = submit_block_receiver.recv().await.expect("channel is open"); let (submitted_hash, submitted_height) = match submit_block_channel_data { MinedBlockEvent::Early { hash, height, .. } - | MinedBlockEvent::Committed { - hash, - height, - early_advertised: false, - } => (hash, height), - event => panic!( - "submitblock should send an authorized early event or the safe committed fallback: {event:?}" - ), + | MinedBlockEvent::Committed { hash, height } => (hash, height), }; assert_eq!(submitted_hash, proposal_block.hash()); assert_eq!(submitted_height, proposal_block.coinbase_height().unwrap()); From 406674b617785709bcc837065fb51ed428c70ba3 Mon Sep 17 00:00:00 2001 From: evan-forbes Date: Thu, 3 Sep 2026 17:42:41 -0500 Subject: [PATCH 20/22] chore(mining): bump public API crate versions --- Cargo.lock | 4 ++-- crates/zakura-consensus/Cargo.toml | 6 +++--- crates/zakura-rpc/Cargo.toml | 8 ++++---- crates/zakura-state/Cargo.toml | 2 +- crates/zakura-utils/Cargo.toml | 2 +- crates/zakurad/Cargo.toml | 8 ++++---- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 059ca18f25..b6927410fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7795,7 +7795,7 @@ dependencies = [ [[package]] name = "zakura-consensus" -version = "7.0.0" +version = "8.0.0" dependencies = [ "blake2b_simd", "chrono", @@ -8297,7 +8297,7 @@ dependencies = [ [[package]] name = "zakura-state" -version = "7.1.0" +version = "8.0.0" dependencies = [ "bincode", "chrono", diff --git a/crates/zakura-consensus/Cargo.toml b/crates/zakura-consensus/Cargo.toml index 460cf956a5..b30a7474ce 100644 --- a/crates/zakura-consensus/Cargo.toml +++ b/crates/zakura-consensus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zakura-consensus" -version = "7.0.0" +version = "8.0.0" authors.workspace = true description = "Implementation of Zcash consensus checks for the Zakura node. Internal crate, published to support cargo install zakura" license.workspace = true @@ -68,7 +68,7 @@ tower-fallback = { package = "zakura-tower-fallback", path = "../tower-fallback/ tower-batch-control = { package = "zakura-tower-batch-control", path = "../tower-batch-control/", version = "1.3.0" } zakura-script = { path = "../zakura-script", version = "3.2.1" } -zakura-state = { path = "../zakura-state", version = "7.0.0" } +zakura-state = { path = "../zakura-state", version = "8.0.0" } zakura-node-services = { path = "../zakura-node-services", version = "3.2.1" } zakura-chain = { path = "../zakura-chain", version = "6.0.0" } zakura-header-chain = { path = "../zakura-header-chain", version = "1.0.0" } @@ -93,7 +93,7 @@ toml = { workspace = true } tokio = { workspace = true, features = ["full", "tracing", "test-util"] } -zakura-state = { path = "../zakura-state", version = "7.0.0", features = ["proptest-impl"] } +zakura-state = { path = "../zakura-state", version = "8.0.0", features = ["proptest-impl"] } zakura-chain = { path = "../zakura-chain", version = "6.0.0", features = ["proptest-impl"] } zakura-test = { path = "../zakura-test/", version = "2.1.0" } diff --git a/crates/zakura-rpc/Cargo.toml b/crates/zakura-rpc/Cargo.toml index 08e6059209..be58faeae5 100644 --- a/crates/zakura-rpc/Cargo.toml +++ b/crates/zakura-rpc/Cargo.toml @@ -111,13 +111,13 @@ zcash_transparent = { workspace = true } zakura-chain = { path = "../zakura-chain", version = "6.0.0", features = [ "json-conversion", ] } -zakura-consensus = { path = "../zakura-consensus", version = "7.0.0" } +zakura-consensus = { path = "../zakura-consensus", version = "8.0.0" } zakura-network = { path = "../zakura-network", version = "7.0.0" } zakura-node-services = { path = "../zakura-node-services", version = "3.2.1", features = [ "rpc-client", ] } zakura-script = { path = "../zakura-script", version = "3.2.1" } -zakura-state = { path = "../zakura-state", version = "7.0.0" } +zakura-state = { path = "../zakura-state", version = "8.0.0" } rustls = { version = "0.23.40", default-features = false, features = ["logging", "ring", "std", "tls12"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "ring", "tls12"] } # Only used to read the validity dates of the configured RPC TLS certificates. @@ -138,13 +138,13 @@ tokio = { workspace = true, features = ["full", "tracing", "test-util"] } zakura-chain = { path = "../zakura-chain", version = "6.0.0", features = [ "proptest-impl", ] } -zakura-consensus = { path = "../zakura-consensus", version = "7.0.0", features = [ +zakura-consensus = { path = "../zakura-consensus", version = "8.0.0", features = [ "proptest-impl", ] } zakura-network = { path = "../zakura-network", version = "7.0.0", features = [ "proptest-impl", ] } -zakura-state = { path = "../zakura-state", version = "7.0.0", features = [ +zakura-state = { path = "../zakura-state", version = "8.0.0", features = [ "proptest-impl", ] } diff --git a/crates/zakura-state/Cargo.toml b/crates/zakura-state/Cargo.toml index e59c81afed..f9709a4873 100644 --- a/crates/zakura-state/Cargo.toml +++ b/crates/zakura-state/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zakura-state" -version = "7.1.0" +version = "8.0.0" authors.workspace = true description = "State contextual verification and storage code for the Zakura node. Internal crate, published to support cargo install zakura" license.workspace = true diff --git a/crates/zakura-utils/Cargo.toml b/crates/zakura-utils/Cargo.toml index 06dbc8fdeb..338bb1a8bb 100644 --- a/crates/zakura-utils/Cargo.toml +++ b/crates/zakura-utils/Cargo.toml @@ -81,7 +81,7 @@ zakura-chain = { path = "../zakura-chain", version = "6.0.0", optional = true } itertools = { workspace = true, optional = true } # This crate is needed for the zakura-checkpoints offline export mode -zakura-state = { path = "../zakura-state", version = "7.0.0", optional = true } +zakura-state = { path = "../zakura-state", version = "8.0.0", optional = true } # This crate is needed for the zakura-checkpoints binary tokio = { workspace = true, features = ["macros", "rt-multi-thread"], optional = true } diff --git a/crates/zakurad/Cargo.toml b/crates/zakurad/Cargo.toml index a8a66f25c2..545e0879f9 100644 --- a/crates/zakurad/Cargo.toml +++ b/crates/zakurad/Cargo.toml @@ -169,13 +169,13 @@ comparison-interpreter = ["zakura-script/comparison-interpreter"] [dependencies] zakura-chain = { path = "../zakura-chain", version = "6.0.0" } -zakura-consensus = { path = "../zakura-consensus", version = "7.0.0" } +zakura-consensus = { path = "../zakura-consensus", version = "8.0.0" } zakura-header-chain = { path = "../zakura-header-chain", version = "1.0.0" } zakura-jsonl-trace = { path = "../zakura-jsonl-trace", version = "1.2.0" } zakura-network = { path = "../zakura-network", version = "7.0.0" } zakura-node-services = { path = "../zakura-node-services", version = "3.2.1", features = ["rpc-client"] } zakura-rpc = { path = "../zakura-rpc", version = "8.0.0" } -zakura-state = { path = "../zakura-state", version = "7.0.0" } +zakura-state = { path = "../zakura-state", version = "8.0.0" } # zakura-script is not used directly, but we list it here to enable the # "comparison-interpreter" feature. (Feature unification will take care of # enabling it in the other imports of zcash-script.) @@ -306,10 +306,10 @@ proptest = { workspace = true } proptest-derive = { workspace = true } zakura-chain = { path = "../zakura-chain", version = "6.0.0", features = ["proptest-impl"] } -zakura-consensus = { path = "../zakura-consensus", version = "7.0.0", features = ["proptest-impl"] } +zakura-consensus = { path = "../zakura-consensus", version = "8.0.0", features = ["proptest-impl"] } zakura-header-chain = { path = "../zakura-header-chain", version = "1.0.0", features = ["test-support"] } zakura-network = { path = "../zakura-network", version = "7.0.0", features = ["proptest-impl", "zakura-testkit"] } -zakura-state = { path = "../zakura-state", version = "7.0.0", features = ["proptest-impl"] } +zakura-state = { path = "../zakura-state", version = "8.0.0", features = ["proptest-impl"] } zakura-test = { path = "../zakura-test", version = "2.1.0" } From f715a592da94e7a7e41a18a2120887e77f0623d2 Mon Sep 17 00:00:00 2001 From: evan-forbes Date: Thu, 3 Sep 2026 17:45:35 -0500 Subject: [PATCH 21/22] chore(rpc): bump major version for mining API --- Cargo.lock | 2 +- crates/zakura-rpc/Cargo.toml | 2 +- crates/zakurad/Cargo.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b6927410fd..f5c98af44d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8165,7 +8165,7 @@ dependencies = [ [[package]] name = "zakura-rpc" -version = "8.0.0" +version = "9.0.0" dependencies = [ "anyhow", "base64 0.22.1", diff --git a/crates/zakura-rpc/Cargo.toml b/crates/zakura-rpc/Cargo.toml index be58faeae5..c36ecf8dd9 100644 --- a/crates/zakura-rpc/Cargo.toml +++ b/crates/zakura-rpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zakura-rpc" -version = "8.0.0" +version = "9.0.0" authors.workspace = true description = "The Zakura node's JSON Remote Procedure Call (JSON-RPC) interface. Internal crate, published to support cargo install zakura" license.workspace = true diff --git a/crates/zakurad/Cargo.toml b/crates/zakurad/Cargo.toml index 545e0879f9..6f58b185ee 100644 --- a/crates/zakurad/Cargo.toml +++ b/crates/zakurad/Cargo.toml @@ -174,7 +174,7 @@ zakura-header-chain = { path = "../zakura-header-chain", version = "1.0.0" } zakura-jsonl-trace = { path = "../zakura-jsonl-trace", version = "1.2.0" } zakura-network = { path = "../zakura-network", version = "7.0.0" } zakura-node-services = { path = "../zakura-node-services", version = "3.2.1", features = ["rpc-client"] } -zakura-rpc = { path = "../zakura-rpc", version = "8.0.0" } +zakura-rpc = { path = "../zakura-rpc", version = "9.0.0" } zakura-state = { path = "../zakura-state", version = "8.0.0" } # zakura-script is not used directly, but we list it here to enable the # "comparison-interpreter" feature. (Feature unification will take care of From e906cee150d9fc2e574d8bcf28c241d7192ce6c6 Mon Sep 17 00:00:00 2001 From: evan-forbes Date: Thu, 3 Sep 2026 18:29:18 -0500 Subject: [PATCH 22/22] ci(semver): patch broken tinyvec release --- .../scripts/patch_tinyvec_for_rustdoc.sh | 44 +++++++++++++++++++ .github/workflows/semver-checks.yml | 11 +++++ 2 files changed, 55 insertions(+) create mode 100755 .github/workflows/scripts/patch_tinyvec_for_rustdoc.sh diff --git a/.github/workflows/scripts/patch_tinyvec_for_rustdoc.sh b/.github/workflows/scripts/patch_tinyvec_for_rustdoc.sh new file mode 100755 index 0000000000..477180f26f --- /dev/null +++ b/.github/workflows/scripts/patch_tinyvec_for_rustdoc.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# tinyvec 1.13.0 omits the alloc::vec macro import in its alloc-only build. +# cargo-semver-checks resolves dependencies outside the workspace lock file, so +# patch both its current and published-baseline builds through an isolated Cargo home. +readonly tinyvec_version=1.13.0 +readonly original_cargo_home="${CARGO_HOME:-$HOME/.cargo}" +patch_root=$(mktemp -d "$RUNNER_TEMP/tinyvec-semver.XXXXXX") +readonly patch_root +readonly patched_cargo_home="$patch_root/cargo-home" +readonly patched_source="$patch_root/tinyvec-$tinyvec_version" + +cargo info "tinyvec@$tinyvec_version" >/dev/null + +mapfile -t source_candidates < <( + find "$original_cargo_home/registry/src" -mindepth 2 -maxdepth 2 -type d \ + -name "tinyvec-$tinyvec_version" +) +if (( ${#source_candidates[@]} != 1 )); then + echo "expected one tinyvec $tinyvec_version source directory, found ${#source_candidates[@]}" >&2 + exit 1 +fi + +cp -a "${source_candidates[0]}" "$patched_source" +readonly source_file="$patched_source/src/tinyvec.rs" +if [[ "$(grep -Fxc 'use alloc::vec::{self, Vec};' "$source_file")" != 1 ]]; then + echo "tinyvec $tinyvec_version no longer matches the expected broken source" >&2 + exit 1 +fi +sed -i 's/use alloc::vec::{self, Vec};/use alloc::{vec, vec::Vec};/' "$source_file" + +mkdir -p "$patched_cargo_home" +ln -s "$original_cargo_home/registry" "$patched_cargo_home/registry" +if [[ -d "$original_cargo_home/git" ]]; then + ln -s "$original_cargo_home/git" "$patched_cargo_home/git" +fi +cat > "$patched_cargo_home/config.toml" <> "$GITHUB_ENV" diff --git a/.github/workflows/semver-checks.yml b/.github/workflows/semver-checks.yml index 939bcc60ce..b7c6aab92f 100644 --- a/.github/workflows/semver-checks.yml +++ b/.github/workflows/semver-checks.yml @@ -39,6 +39,7 @@ on: - "**/Cargo.lock" - .github/workflows/semver-checks.yml - .github/workflows/scripts/affected_semver_packages.py + - .github/workflows/scripts/patch_tinyvec_for_rustdoc.sh - .github/workflows/scripts/test_affected_semver_packages.py push: branches: [main] @@ -48,6 +49,7 @@ on: - "**/Cargo.lock" - .github/workflows/semver-checks.yml - .github/workflows/scripts/affected_semver_packages.py + - .github/workflows/scripts/patch_tinyvec_for_rustdoc.sh - .github/workflows/scripts/test_affected_semver_packages.py # Run in the merge queue so queued changes are revalidated against the latest @@ -202,6 +204,9 @@ jobs: - uses: ./.github/actions/setup-zakura-build + - name: Patch tinyvec for alloc-only rustdoc builds + run: .github/workflows/scripts/patch_tinyvec_for_rustdoc.sh + - name: Check package against its stable crates.io baseline env: PACKAGE: ${{ matrix.package }} @@ -252,6 +257,9 @@ jobs: - uses: ./.github/actions/setup-zakura-build + - name: Patch tinyvec for alloc-only rustdoc builds + run: .github/workflows/scripts/patch_tinyvec_for_rustdoc.sh + - name: Install nightly rustdoc toolchain run: rustup toolchain install nightly --profile minimal @@ -299,6 +307,9 @@ jobs: - uses: ./.github/actions/setup-zakura-build + - name: Patch tinyvec for alloc-only rustdoc builds + run: .github/workflows/scripts/patch_tinyvec_for_rustdoc.sh + - name: Check all crates and warm published release baselines id: warm env: