diff --git a/.gitignore b/.gitignore index c31aa4f..a70e78c 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ out/ playground/ .data/ /target +.* diff --git a/Cargo.lock b/Cargo.lock index 7928d95..5500447 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5930,6 +5930,7 @@ dependencies = [ "bytes", "futures-core", "futures-sink", + "futures-util", "pin-project-lite", "tokio", ] diff --git a/Cargo.toml b/Cargo.toml index 5132246..60e3eba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,10 +8,10 @@ description = "Extract blockchain data in JSON format and archive it as Avro fil [dependencies] clap = { version = "4.6", features = ["derive", "string"] } serde = { version = "1.0" , features = ["derive"] } -serde_json = "1.0" +serde_json = { version = "1.0", features = ["raw_value"] } tokio = { version = "1.50", features = ["fs", "rt-multi-thread", "macros"] } tokio-retry2 = { version = "0.9", features = ["jitter"] } -tokio-util = { version = "0.7" , features = ["io", "io-util"] } +tokio-util = { version = "0.7" , features = ["io", "io-util", "rt"] } tonic = { version = "0.14", features = ["tls-ring", "tls-native-roots", "gzip"] } futures = "0.3" async-trait = "0.1" diff --git a/README.adoc b/README.adoc index 6dcea50..4b12a31 100644 --- a/README.adoc +++ b/README.adoc @@ -87,6 +87,16 @@ Options: --aws.trust-tls Trust any TLS certificate for AWS / S3 (default is false) + --stream.url + Publish stream data to a broker at the given URL. The scheme selects the backend: + + - `pulsar://HOST:PORT` — Apache Pulsar. + + Selecting a streaming target restricts the run to the `stream` command — `archive`, `fix`, `verify`, and `compact` are rejected at startup because topics are append-only. + + --stream.topics + Prefix used to build the per-field topic names. Each field is published to `-` (e.g. `-blocks`, `-tx-json`). For Pulsar, include the full topic path up to the prefix, e.g. `persistent://public/default/archive-eth` + -d, --dir Target directory @@ -112,10 +122,21 @@ Options: [Fix Command] Set to remove any existing data in whole chunk if any of tables is missing a block in the chunk or has broken values. Default is `false`, which deleted only tables with missing / corrupted data --compression - Compression algorithm to use when writing new Avro files. Default is `zstd` + Compression algorithm to use for new output. For `--format=avro` it's the Avro file codec; for the Pulsar streaming target it's the producer compression applied to every message. Default is `zstd` [possible values: snappy, zstd] + --retry + Retry policy for transient blockchain fetch failures. + + - `bounded` — give up after a fixed number of attempts (current behaviour for file targets). - `forever` — keep retrying indefinitely with exponential backoff. Required for ordered streaming targets where a missed record breaks the topic-order contract. + + Defaults: `forever` for streaming-ordered targets (e.g. Pulsar), `bounded` otherwise. + + Possible values: + - bounded: Give up after a fixed number of attempts. Default for file targets, where a failed fetch can be repaired later by the `fix` command + - forever: Keep retrying indefinitely. Default for streaming-ordered targets (Pulsar) where a gap breaks the topic-order contract permanently + --follow [Stream Command] Follow mode for new blocks: `latest` - follow the latest blocks (default); `finalized` - follow only finalized blocks @@ -142,8 +163,6 @@ Options: -V, --version Print version - - ---- === Commands @@ -163,7 +182,7 @@ One for blocks in that range, and another one with _all_ transactions in all blo See <>. -==== Stream +==== Stream as Tables Continuously append fresh blocks one by one to the archive. In addition to the copying, Dshackle archive can be configured to notify an external system about new blocks in the archive. @@ -181,6 +200,37 @@ To notify an external system, there are two options: See <>. +==== Stream as Records + +When `--stream.url` passed Dshackle Archive publishes stream data to a broker, such as Apache Pulsar, at the given URL. +Topic receives each individual record in a block. I.e., instead of writing a large Avro file like in "_Stream as Tables_" mode, this mode produces a new message for each transaction in a block. + +`--stream.topics` specifies the prefix used to build the per-field topic names. +Each field is published to `-` (e.g. `-blocks`, `-tx-json`). + +.Topics used if prefix specified as `ethereum`: +- `ethereum-blocks` +- `ethereum-uncles` +- `ethereum-tx-json` +- `ethereum-tx-raw` +- `ethereum-tx-receipt` +- `ethereum-trace-json` +- `ethereum-trace-statediff` + +.The JSON format is: +- `table` - `blocks`, `transactions` or `traces` +- `field` - the type of message, e.g. `block`, "uncles", `tx`, "raw", "receipt", "calls", `statediff` +- `blockchain` - what was specified in `--blockchain` option, e.g. `ETH` or `BTC` +- `timestamp` - Block timestamp as reported by the node, serialized as RFC 3339 +- `height` - Block height, as number +- `blockId` - Block hash +- `parentId` - Parent block hash +- `txCount` - Number of transactions in the block +- `txIndex` - Transaction index in block. Present on tx/trace records +- `txId` - Transaction hash. Present on tx/trace records +- `uncleIndex` - Uncle index. Present on Ethereum uncle messages only +- `value` - Original data (JSON or HEX string for raw transaction) + ==== Compact Merge individual block files into larger range files. diff --git a/src/archiver/archiver.rs b/src/archiver/archiver.rs index 232efd7..b9b2a5b 100644 --- a/src/archiver/archiver.rs +++ b/src/archiver/archiver.rs @@ -3,15 +3,16 @@ use std::sync::Arc; use async_trait::async_trait; use chrono::Utc; use tokio::sync::mpsc::Sender; +use tokio_util::sync::CancellationToken; use crate::blockchain::{BlockchainData, BlockchainTypes}; use crate::archiver::datakind::{DataKind, DataOptions}; +use crate::archiver::ProcessOutcome; use crate::notify::empty::EmptyNotifier; use crate::notify::{Maturity, Notification, Notifier, RunMode}; use crate::archiver::range::{Height, Range}; use crate::global; use crate::storage::WriteTarget; -#[derive(Clone)] pub struct Archiver { b: PhantomData, pub target: Arc, @@ -19,6 +20,20 @@ pub struct Archiver { pub notifications: Sender, } +// Manual `Clone` impl: all fields are reference-counted, so cloning works +// without requiring `B: Clone` or `TS: Clone` (which derive(Clone) would +// otherwise demand). +impl Clone for Archiver { + fn clone(&self) -> Self { + Self { + b: PhantomData, + target: self.target.clone(), + data_provider: self.data_provider.clone(), + notifications: self.notifications.clone(), + } + } +} + impl Archiver { pub fn new_simple(target: Arc, data_provider: Arc) -> Self { @@ -43,14 +58,35 @@ impl Archiver { } +/// Drives a full archive run for a single height or a range of heights. +/// +/// The `cancel` token lets the caller abort cooperatively — it's wired into +/// every parallel fetcher and into the ordering layer, so a fire mid-run +/// drops in-flight RPCs and abandons partial sinks without raising a +/// "closed with gap" error. Non-cancellable callers (historical archive, +/// finalized streams) pass a fresh, never-fired token. #[async_trait] pub trait ArchiveAll { - async fn archive(&self, what: T, mode: RunMode, maturity: Option, options: &DataOptions) -> anyhow::Result<()>; + async fn archive( + &self, + what: T, + mode: RunMode, + maturity: Option, + options: &DataOptions, + cancel: &CancellationToken, + ) -> anyhow::Result<()>; } #[async_trait] impl ArchiveAll for Archiver { - async fn archive(&self, what: Height, mode: RunMode, maturity: Option, options: &DataOptions) -> anyhow::Result<()> { + async fn archive( + &self, + what: Height, + mode: RunMode, + maturity: Option, + options: &DataOptions, + cancel: &CancellationToken, + ) -> anyhow::Result<()> { let start_time = Utc::now(); let notification = Notification { @@ -68,44 +104,98 @@ impl ArchiveAll for Archiver location: "".to_string(), }; - let blocks = self.process_blocks(Range::Single(what.clone()), notification.clone(), options).await?; + let (blocks, blocks_notif) = match self + .process_blocks(Range::Single(what.clone()), notification.clone(), options, cancel) + .await? + { + ProcessOutcome::Completed { value, notification } => (value, notification), + ProcessOutcome::Cancelled => { + tracing::info!( + "Block {} cancelled (re-org) — skipping tx/trace fetch", + what + ); + return Ok(()); + } + }; let range = Range::Single(what.clone()); - let (success_tx, success_trace) = tokio::join! { - async { + // Run txes and traces concurrently. Each branch returns either: + // - Some(ProcessOutcome::Completed { .. } | ProcessOutcome::Cancelled) + // when the side was actually run + // - None when the side errored — we log inline, surface the error + // state to the joint, but don't abort the other side + // - Some(Completed { value: (), notification: None }) when the + // side is disabled by DataOptions (treated as success-with-nothing) + let (tx_side, trace_side) = tokio::join! { + async { if options.include_tx() { - if let Err(e) = self.process_txes(range.clone(), notification.clone(), &blocks, options).await { - tracing::warn!("Failed to archive txes for block {}: {}", what, e); - false - } else { - true + match self.process_txes(range.clone(), notification.clone(), &blocks, options, cancel).await { + Ok(outcome) => Some(outcome), + Err(e) => { + tracing::warn!("Failed to archive txes for block {}: {}", what, e); + None + } } } else { - true + Some(ProcessOutcome::Completed { value: (), notification: None }) } }, async { if options.include_trace() { - if let Err(e) = self.process_traces(range.clone(), notification.clone(), &blocks, options).await { - tracing::warn!("Failed to archive traces for block {}: {}", what, e); - false - } else { - true + match self.process_traces(range.clone(), notification.clone(), &blocks, options, cancel).await { + Ok(outcome) => Some(outcome), + Err(e) => { + tracing::warn!("Failed to archive traces for block {}: {}", what, e); + None + } } } else { - true + Some(ProcessOutcome::Completed { value: (), notification: None }) } } }; - let success = success_tx & success_trace; let duration = Utc::now().signed_duration_since(start_time); let duration_secs = duration.num_milliseconds() as f64 / 1000.0; crate::metrics::observe_block_archive(duration_secs); - if success { - tracing::info!("Blocks {} is archived in {}ms", what, duration.num_milliseconds()); + + let cancelled = tx_side.as_ref().map_or(false, |o| o.is_cancelled()) + || trace_side.as_ref().map_or(false, |o| o.is_cancelled()); + let errored = tx_side.is_none() || trace_side.is_none(); + + // Publish notifications only when neither side cancelled — otherwise + // a torn notification stream (blocks notified, txes/traces silent) + // would mislead consumers into thinking the archive is incomplete + // due to corruption rather than a re-org. On cancel the partial + // artifacts have already been cleaned up via writer Drop in the + // process_* functions; the doomed block is invisible to consumers. + if !cancelled && !global::get_shutdown().is_signalled() { + self.publish_notifications([ + blocks_notif, + extract_notification(tx_side.as_ref()), + extract_notification(trace_side.as_ref()), + ]) + .await; + } + + if cancelled { + tracing::info!( + "Block {} cancelled (re-org) in {}ms", + what, + duration.num_milliseconds() + ); + } else if errored { + tracing::warn!( + "Blocks {} is partially archived (with error) in {}ms", + what, + duration.num_milliseconds() + ); } else { - tracing::warn!("Blocks {} is partially archived (with error) in {}ms", what, duration.num_milliseconds()); + tracing::info!( + "Blocks {} is archived in {}ms", + what, + duration.num_milliseconds() + ); } Ok(()) } @@ -113,7 +203,14 @@ impl ArchiveAll for Archiver #[async_trait] impl ArchiveAll for Archiver { - async fn archive(&self, what: Range, mode: RunMode, maturity: Option, options: &DataOptions) -> anyhow::Result<()> { + async fn archive( + &self, + what: Range, + mode: RunMode, + maturity: Option, + options: &DataOptions, + cancel: &CancellationToken, + ) -> anyhow::Result<()> { let start_time = Utc::now(); tracing::debug!("Archiving range: {}", what); @@ -132,40 +229,71 @@ impl ArchiveAll for Archiver location: "".to_string(), }; - let blocks = self.process_blocks(what.clone(), notification.clone(), options).await?; + let (blocks, blocks_notif) = match self + .process_blocks(what.clone(), notification.clone(), options, cancel) + .await? + { + ProcessOutcome::Completed { value, notification } => (value, notification), + ProcessOutcome::Cancelled => { + tracing::info!(range = %what, "Range archive cancelled before tx/trace fetch"); + return Ok(()); + } + }; let (result_tx, result_trace) = tokio::join!( async { if options.include_tx() { tracing::debug!(range = %what, "Process txes"); - self.process_txes(what.clone(), notification.clone(), &blocks, options).await + self.process_txes(what.clone(), notification.clone(), &blocks, options, cancel).await } else { - Ok(()) + Ok(ProcessOutcome::Completed { value: (), notification: None }) } }, async { if options.include_trace() { tracing::debug!(range = %what, "Process traces"); - self.process_traces(what.clone(), notification.clone(), &blocks, options).await + self.process_traces(what.clone(), notification.clone(), &blocks, options, cancel).await } else { - Ok(()) + Ok(ProcessOutcome::Completed { value: (), notification: None }) } } ); - result_tx?; - result_trace?; + // Surface both errors when both sides fail — `?` on the first would + // silently drop the second. Operators investigating a stuck Range + // archive need both messages. + let (tx_outcome, trace_outcome) = match (result_tx, result_trace) { + (Ok(tx), Ok(trace)) => (tx, trace), + (Err(tx_err), Err(trace_err)) => { + tracing::warn!(range = %what, "process_traces failed: {}", trace_err); + return Err(tx_err); + } + (Err(tx_err), Ok(_)) => return Err(tx_err), + (Ok(_), Err(trace_err)) => return Err(trace_err), + }; let shutdown = global::get_shutdown(); if shutdown.is_signalled() { return Ok(()); } + let cancelled = tx_outcome.is_cancelled() || trace_outcome.is_cancelled(); + if !cancelled { + self.publish_notifications([ + blocks_notif, + extract_notification(Some(&tx_outcome)), + extract_notification(Some(&trace_outcome)), + ]) + .await; + } + let duration = Utc::now().signed_duration_since(start_time); if what.len() == 1 { let duration_secs = duration.num_milliseconds() as f64 / 1000.0; crate::metrics::observe_block_archive(duration_secs); } - if duration.num_seconds() > 2 { + if cancelled { + tracing::info!(range = %what, "Range archive cancelled mid-run"); + } else if duration.num_seconds() > 2 { tracing::info!("Range {} is archived in {}sec", what, duration.num_seconds()); } else { tracing::info!("Range {} is archived in {}ms", what, duration.num_milliseconds()); @@ -174,3 +302,34 @@ impl ArchiveAll for Archiver Ok(()) } } + +/// Extract the deferred [`Notification`] from a [`ProcessOutcome`], if any. +/// +/// Returns `None` when the side errored (outer Option is None), was +/// cancelled, or completed without producing a notification (target skipped +/// an existing file). The archiver coordinator collects these across all +/// three process_* calls and publishes them as a batch — only when the +/// whole run is known to be uncancelled. +fn extract_notification(side: Option<&ProcessOutcome>) -> Option { + match side { + Some(ProcessOutcome::Completed { + notification: Some(n), + .. + }) => Some(n.clone()), + _ => None, + } +} + +impl Archiver { + /// Publish the queued notifications from a completed run. Errors from + /// the notification channel are logged but not propagated — the data + /// itself has already landed in the archive; a failed notify shouldn't + /// fail the whole run. + async fn publish_notifications(&self, notifs: impl IntoIterator>) { + for notif in notifs.into_iter().flatten() { + if let Err(e) = self.notifications.send(notif).await { + tracing::warn!("Failed to publish notification: {}", e); + } + } + } +} diff --git a/src/archiver/block.rs b/src/archiver/block.rs index 76a4b31..502d540 100644 --- a/src/archiver/block.rs +++ b/src/archiver/block.rs @@ -3,8 +3,10 @@ use anyhow::anyhow; use chrono::Utc; use tokio::sync::Semaphore; use tokio::task::JoinSet; +use tokio_util::sync::CancellationToken; use crate::archiver::archiver::Archiver; -use crate::archiver::BlockTransactions; +use crate::archiver::order::AppendSink; +use crate::archiver::{BlockTransactions, ProcessOutcome}; use crate::blockchain::{BlockReference, BlockchainData, BlockchainTypes}; use crate::archiver::datakind::{DataKind, DataOptions}; use crate::notify::Notification; @@ -17,10 +19,23 @@ impl Archiver { /// /// Archive the blocks and return all the blocks in that the archive for reference in other tables. /// Results are sorted by height regardless of fetch order. - pub async fn process_blocks(&self, blocks: Range, notification: Notification, options: &DataOptions) -> anyhow::Result> { + /// + /// The `cancel` token lets the caller abandon the run mid-flight — used by + /// live streaming when the re-org follower learns the block has been + /// replaced. Non-streaming callers pass a fresh, never-cancelled token. + pub async fn process_blocks( + &self, + blocks: Range, + notification: Notification, + options: &DataOptions, + cancel: &CancellationToken, + ) -> anyhow::Result>> { let shutdown = global::get_shutdown(); if shutdown.is_signalled() { - return Ok(vec![]); + return Ok(ProcessOutcome::Completed { + value: vec![], + notification: None, + }); } let dry_run = global::is_dry_run(); let file = self.target.create(DataKind::Blocks, &blocks, options.overwrite) @@ -32,30 +47,54 @@ impl Archiver { } let file_url = file.as_ref().map(|f| f.get_url()); let file = file.map(Arc::new); + // Order block appends by height when the target requires it + // (streaming backends). For file backends `needs_ordering()` is false + // and the sink falls through to a thin pass-through that hands rows + // to the writer in arrival order, without the channel/buffer hop. + let ordered = self.target.needs_ordering(); + let sink = file + .clone() + .map(|f| Arc::new(AppendSink::new(f, blocks.start(), ordered))); let mut jobs = JoinSet::new(); let semaphore = Arc::new(Semaphore::new(global::get_threads().blocks)); for height in blocks.iter_height().collect::>() { let provider = self.data_provider.clone(); - let file = file.clone(); + let sink = sink.clone(); let shutdown = shutdown.clone(); let semaphore = semaphore.clone(); let block_height = height.height; + let cancel = cancel.clone(); jobs.spawn(async move { if shutdown.is_signalled() { return Ok(None); } let _permit = semaphore.acquire().await.unwrap(); - let block_ref = BlockReference::Height(height); - let (record, block, txes) = provider.fetch_block(&block_ref).await?; - if !dry_run { - if let Some(file) = &file { - file.append(record).await?; + // Prefer a hash-pinned lookup when the follower gave us one, + // so we don't race a re-org between subscription and fetch. + // Falls back to height-only when no hash is present (e.g., + // batch archive over a numeric range). See the + // `From for BlockReference` impl in blockchain/mod.rs. + let block_ref: BlockReference = height.into(); + let work = async { + let (record, block, txes) = provider.fetch_block(&block_ref).await?; + if !dry_run { + if let Some(sink) = &sink { + sink.append_at(block_height, record).await?; + } } + crate::progress::on_record(); + crate::metrics::add_items(&DataKind::Blocks, crate::metrics::Direction::Write, 1); + Ok::<_, anyhow::Error>(Some((block_height, block, txes))) + }; + // Race the fetch against the cancel signal. On cancel the + // task returns `None`, the drain loop counts it as a no-data + // result, and the outer function inspects `cancel.is_cancelled()` + // after the drain to decide whether to abandon or close. + tokio::select! { + _ = cancel.cancelled() => Ok(None), + r = work => r, } - crate::progress::on_record(); - crate::metrics::add_items(&DataKind::Blocks, crate::metrics::Direction::Write, 1); - Ok::<_, anyhow::Error>(Some((block_height, block, txes))) }); } @@ -66,30 +105,66 @@ impl Archiver { results.push(entry); } } + + // Cancel-aware shutdown: skip the commit + notification path entirely + // when the run was abandoned. `abandon` aborts the ordering layer's + // drain task so any rows still buffered for the doomed block don't + // raise a "closed with a gap" error. The file is intentionally NOT + // closed: file-backend Drop impls remove the partial artifact (so a + // subsequent re-org replacement with `overwrite=false` isn't blocked + // by an empty/half-written file), and streaming-broker `close` is a + // no-op anyway since messages have already left the producer. + if cancel.is_cancelled() { + if !dry_run { + if let Some(sink) = sink { + let sink = Arc::into_inner(sink) + .ok_or_else(|| anyhow!("AppendSink still referenced after all tasks completed"))?; + sink.abandon().await?; + } + // Drop the writer Arc unconditionally — its Drop impl deletes + // the partial file on disk (FsFileWriter / JsonFsWriter) or + // abandons the S3 multipart upload (ObjectsStorage). For + // Pulsar the drop is a no-op. + drop(file); + } + return Ok(ProcessOutcome::Cancelled); + } + results.sort_by_key(|(height, _, _)| *height); let results: BlockTransactions = results.into_iter() .map(|(_, block, txes)| (block, txes)) .collect(); if !dry_run { + // Close the ordering layer first — it drains any buffered rows + // into the underlying writer (no-op for the direct variant). Only + // then is it safe to take the sole reference to the writer and + // close it. + if let Some(sink) = sink { + let sink = Arc::into_inner(sink) + .ok_or_else(|| anyhow!("AppendSink still referenced after all tasks completed"))?; + sink.close().await?; + } if let Some(file) = file { let file = Arc::into_inner(file) .ok_or_else(|| anyhow!("File writer still referenced after all tasks completed"))?; let _ = file.close().await?; } } - if let Some(file_url) = file_url { - let notification_tx = Notification { - file_type: DataKind::Blocks, - location: file_url, - ts: Utc::now(), - - ..notification - }; - if !dry_run { - let _ = self.notifications.send(notification_tx).await; - } - } - Ok(results) + // Build the notification but defer the send to `archive()`. If a + // concurrent process_txes / process_traces ends up cancelled, the + // archiver will discard all queued notifications atomically so the + // consumer never sees a torn notification stream for the doomed + // block. + let notification = file_url.map(|file_url| Notification { + file_type: DataKind::Blocks, + location: file_url, + ts: Utc::now(), + ..notification + }); + Ok(ProcessOutcome::Completed { + value: results, + notification, + }) } } diff --git a/src/archiver/datakind.rs b/src/archiver/datakind.rs index e3d9585..cc0ad8d 100644 --- a/src/archiver/datakind.rs +++ b/src/archiver/datakind.rs @@ -19,14 +19,29 @@ pub enum DataKind { } impl DataKind { - /// Label value used in Prometheus metrics (the `type` tag). - pub fn metrics_label(&self) -> &'static str { + /// Short, singular label for one record of this kind. Used as the + /// Prometheus metrics `type` tag (one row = one "block" / "transaction" + /// / "trace"). For the *collection* name (a topic, a table) use + /// [`DataKind::table`] instead. + pub fn label(&self) -> &'static str { match self { DataKind::Blocks => "block", DataKind::Transactions => "transaction", DataKind::TransactionTraces => "trace", } } + + /// Plural collection name for this data kind. Used as the `table` + /// field on the streaming envelope and matches the canonical Avro / + /// filesystem table names (`blocks.avro`, `txes` directory, …). + /// Singular per-record contexts use [`DataKind::label`]. + pub fn table(&self) -> &'static str { + match self { + DataKind::Blocks => "blocks", + DataKind::Transactions => "transactions", + DataKind::TransactionTraces => "traces", + } + } } impl FromStr for DataKind { diff --git a/src/archiver/mod.rs b/src/archiver/mod.rs index a397e6a..cf72763 100644 --- a/src/archiver/mod.rs +++ b/src/archiver/mod.rs @@ -7,12 +7,52 @@ pub mod filenames; pub mod range_bag; pub mod blocks_config; pub mod range_group; +pub mod resume; +pub mod order; pub use archiver::{ArchiveAll, Archiver}; +pub use resume::{ScanResume, StreamResume}; +pub use order::{AppendSink, OrderedSink}; use crate::blockchain::BlockchainTypes; +use crate::notify::Notification; #[allow(type_alias_bounds)] pub type BlockTransactions = Vec<(B::BlockParsed, Vec)>; pub type BlockHash = String; + +/// Result of an archiver process step that can be cleanly cut short by +/// upstream cancellation (e.g., a re-org invalidating the block being +/// fetched). +/// +/// `Completed { value, notification }` is the normal outcome — work +/// finished and produced the inner value, plus an optional notification +/// the caller should publish once the entire run is known to be +/// uncancelled. Notifications are NOT sent inside the process step itself: +/// otherwise a concurrent `process_txes` and `process_traces` could race +/// the cancellation signal and emit a torn notification stream (one side +/// publishes before observing the cancel, the other observes it and +/// stays silent), leaving downstream consumers with partial-block state +/// they can't distinguish from corruption. +/// +/// `Cancelled` means the run was abandoned cooperatively; any +/// partially-written rows are dropped and the file is left to Drop (which +/// deletes it for file backends and is a no-op for streaming brokers +/// whose messages have already landed on the wire). Real errors continue +/// to flow through `Result::Err`, so callers' `?` still abort on genuine +/// fetch / storage failures. +#[derive(Debug)] +pub enum ProcessOutcome { + Completed { + value: T, + notification: Option, + }, + Cancelled, +} + +impl ProcessOutcome { + pub fn is_cancelled(&self) -> bool { + matches!(self, ProcessOutcome::Cancelled) + } +} diff --git a/src/archiver/order.rs b/src/archiver/order.rs new file mode 100644 index 0000000..c576a24 --- /dev/null +++ b/src/archiver/order.rs @@ -0,0 +1,489 @@ +// Copyright 2026 EmeraldPay Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. + +//! Per-writer ordering layer for parallel fetches. +//! +//! The archiver fetches blocks, transactions, and traces in parallel +//! (`JoinSet` + a semaphore), then forwards each result to a +//! [`crate::storage::TargetFileWriter`] as soon as the fetch finishes. For +//! file-based targets (Avro, JSON) the order of `append` calls doesn't matter — +//! the file just collects records. For streaming targets (Pulsar today, Kafka +//! next) the broker preserves messages in *publish* order, so the order in +//! which we hand records to the writer becomes the order consumers see. +//! +//! [`OrderedSink`] sits between the parallel fetchers and the writer: each +//! fetcher submits its row tagged with a logical index, and a single drain +//! task forwards rows to the underlying writer in ascending index order. Rows +//! that arrive before their predecessor are buffered (not "until the end" — +//! only until the missing earlier index lands) so the first in-order run is +//! released as early as possible. +//! +//! Indices are chain-natural: +//! +//! - For blocks within a range, the index is the block height. +//! - For transactions / traces within a range, the index is a flat +//! `(block_position, tx_index)` ordinal so that block N's txes are all +//! emitted before block N+1's, and within each block they go in +//! `tx_index` order. +//! +//! The sink reports an error if the channel closes with any "future" rows +//! still pending — i.e., a fetcher silently dropped its row and the sequence +//! has a gap. + +use std::collections::BTreeMap; +use std::sync::Arc; + +use anyhow::{anyhow, Result}; +use tokio::sync::mpsc; +use tokio::task::JoinHandle; + +use crate::record::ArchiveRow; +use crate::storage::TargetFileWriter; + +/// Dispatching sink used by the archiver fan-out sites. +/// +/// Targets that need strict in-order delivery +/// (see [`crate::storage::WriteTarget::needs_ordering`]) get the +/// [`OrderedSink`] path with its drain task and buffer; targets that don't +/// (file backends) get a thin pass-through that calls `writer.append` +/// directly, avoiding the per-row channel hop. Both variants expose the same +/// `append_at(idx, row)` API so the call site stays uniform. +pub enum AppendSink { + Ordered(OrderedSink), + /// Pass-through. `append_at`'s `idx` is ignored — the writer accepts rows + /// in whatever order the parallel fetchers complete. + Direct(Arc), +} + +impl AppendSink { + /// Build the appropriate sink variant. `ordered` typically comes from + /// [`crate::storage::WriteTarget::needs_ordering`]; `start_index` is the + /// first valid index in the ordered case and unused otherwise. + pub fn new(writer: Arc, start_index: u64, ordered: bool) -> Self { + if ordered { + AppendSink::Ordered(OrderedSink::new(writer, start_index)) + } else { + AppendSink::Direct(writer) + } + } + + /// Submit `row` at logical position `index`. In [`AppendSink::Ordered`] + /// mode the row may be buffered until earlier indices land; in + /// [`AppendSink::Direct`] mode it is forwarded to the writer immediately. + pub async fn append_at(&self, index: u64, row: ArchiveRow) -> Result<()> { + match self { + AppendSink::Ordered(s) => s.append_at(index, row).await, + AppendSink::Direct(w) => w.append(row).await, + } + } + + /// Finalize the sink. For [`AppendSink::Ordered`] this awaits the drain + /// task and surfaces any gap/duplicate errors. For [`AppendSink::Direct`] + /// it drops the inner `Arc` clone so the caller's outer Arc is the + /// sole reference for the file `close()`. + pub async fn close(self) -> Result<()> { + match self { + AppendSink::Ordered(s) => s.close().await, + AppendSink::Direct(_) => Ok(()), + } + } + + /// Abandon the sink: stop the drain task immediately, drop any pending + /// rows without raising a gap error. Use this when the upstream work was + /// cancelled (e.g. a re-org invalidated the block) — completing the run + /// is meaningless, and the unfilled gap is expected, not an error. + pub async fn abandon(self) -> Result<()> { + match self { + AppendSink::Ordered(s) => s.abandon().await, + AppendSink::Direct(_) => Ok(()), + } + } +} + +/// Ordering wrapper around a [`TargetFileWriter`]. +/// +/// Construction spawns a background drain task that owns an `Arc` and +/// forwards rows in strict ascending order of the submitted index. Cloning +/// the sink (via `Arc::clone`) hands the channel sender to additional +/// producers; closing it (via [`OrderedSink::close`]) flushes the buffer and +/// propagates any error from the drain task. +pub struct OrderedSink { + tx: mpsc::Sender<(u64, ArchiveRow)>, + handle: JoinHandle>, +} + +impl OrderedSink { + /// Wrap `writer` in an ordering layer that releases rows starting at + /// `start_index` and counts up by one for each consecutive entry. + /// + /// Submissions arrive over a bounded channel — large enough to absorb the + /// burst from a typical block's worth of concurrent fetches without + /// stalling, small enough to apply backpressure if the writer is slow. + pub fn new(writer: Arc, start_index: u64) -> Self + where + W: TargetFileWriter + Send + Sync + 'static, + { + let (tx, mut rx) = mpsc::channel::<(u64, ArchiveRow)>(32); + let handle = tokio::spawn(async move { + let mut next = start_index; + let mut pending: BTreeMap = BTreeMap::new(); + while let Some((idx, row)) = rx.recv().await { + if idx < next { + return Err(anyhow!( + "OrderedSink received index {} below cursor {} (already forwarded)", + idx, + next + )); + } + if pending.insert(idx, row).is_some() { + return Err(anyhow!( + "OrderedSink received duplicate index {}", + idx + )); + } + // Drain whatever consecutive run is now available at the head. + while let Some(row) = pending.remove(&next) { + writer.append(row).await?; + next += 1; + } + } + if !pending.is_empty() { + let smallest = pending.keys().next().copied().unwrap_or(0); + return Err(anyhow!( + "OrderedSink closed with a gap: cursor at {}, {} row(s) pending starting at {}", + next, + pending.len(), + smallest + )); + } + Ok(()) + }); + Self { tx, handle } + } + + /// Submit `row` at logical position `index`. If `index` is the next + /// expected one it is forwarded to the writer immediately; otherwise it + /// is buffered until its predecessor arrives. + pub async fn append_at(&self, index: u64, row: ArchiveRow) -> Result<()> { + self.tx + .send((index, row)) + .await + .map_err(|_| anyhow!("OrderedSink drain task has ended")) + } + + /// Close the sink: drop the sender, await the drain task, propagate any + /// inner-writer or gap errors. Must be called for clean shutdown. + pub async fn close(self) -> Result<()> { + drop(self.tx); + self.handle + .await + .map_err(|e| anyhow!("OrderedSink drain task panicked: {}", e))? + } + + /// Abandon the sink: abort the drain task and discard any pending rows. + /// + /// Unlike [`close`](Self::close), this masks the "closed with gap" + /// diagnostic — the caller is acknowledging that the run was cut short + /// (typically because a re-org invalidated the block) and that whatever + /// is still buffered is now meaningless. Real writer failures that + /// surfaced before abandon was called are still propagated so the caller + /// can distinguish "cancelled cleanly" from "broker rejected publish". + pub async fn abandon(self) -> Result<()> { + drop(self.tx); + self.handle.abort(); + match self.handle.await { + // Drain finished naturally before abort took effect. The Result it + // returned reflects whether the underlying writer succeeded — if a + // prior append failed we must surface it; the gap diagnostic is + // already masked because `drop(self.tx)` lets the drain loop exit + // cleanly. + Ok(Ok(())) => Ok(()), + Ok(Err(writer_err)) => Err(writer_err), + // Drain task was actually aborted mid-iteration; that's the + // intended outcome of `abandon`. + Err(je) if je.is_cancelled() => Ok(()), + Err(je) => Err(anyhow!("OrderedSink drain task panicked: {}", je)), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Mutex; + + use async_trait::async_trait; + use chrono::{TimeZone, Utc}; + + use crate::archiver::datakind::DataKind; + use crate::record::{ArchiveRow, BlockchainType, Field}; + use crate::storage::{TargetFile, TargetFileWriter}; + + /// Recording writer: appends are captured in a Mutex so tests can + /// assert the order rows reach the writer regardless of submission order. + struct Recorder { + appended: Arc>>, + } + + impl TargetFile for Recorder { + fn get_url(&self) -> String { + "test://recorder".to_string() + } + } + + #[async_trait] + impl TargetFileWriter for Recorder { + async fn append(&self, row: ArchiveRow) -> Result<()> { + self.appended.lock().unwrap().push(row.height); + Ok(()) + } + async fn close(self) -> Result<()> { + Ok(()) + } + } + + fn row(height: u64) -> ArchiveRow { + ArchiveRow { + kind: DataKind::Blocks, + blockchain_type: BlockchainType::Ethereum, + blockchain_id: "X".to_string(), + archive_ts: Utc::now(), + height, + block_id: format!("0xB{}", height), + timestamp: Utc.timestamp_millis_opt(0).unwrap(), + parent_id: None, + tx_index: None, + tx_id: None, + tx_count: None, + fields: vec![Field::BlockJson(b"x".to_vec())], + } + } + + #[tokio::test] + async fn forwards_in_order_when_submissions_are_already_in_order() { + let appended = Arc::new(Mutex::new(Vec::new())); + let writer = Arc::new(Recorder { + appended: appended.clone(), + }); + let sink = OrderedSink::new(writer, 0); + for i in 0..5 { + sink.append_at(i, row(i)).await.unwrap(); + } + sink.close().await.unwrap(); + assert_eq!(*appended.lock().unwrap(), vec![0, 1, 2, 3, 4]); + } + + #[tokio::test] + async fn buffers_out_of_order_and_releases_when_gap_fills() { + let appended = Arc::new(Mutex::new(Vec::new())); + let writer = Arc::new(Recorder { + appended: appended.clone(), + }); + let sink = OrderedSink::new(writer, 0); + // Submit out of order: 2, 0, 3, 1, 4 + sink.append_at(2, row(2)).await.unwrap(); + sink.append_at(0, row(0)).await.unwrap(); + sink.append_at(3, row(3)).await.unwrap(); + sink.append_at(1, row(1)).await.unwrap(); + sink.append_at(4, row(4)).await.unwrap(); + sink.close().await.unwrap(); + // Writer must have seen them in strict ascending order. + assert_eq!(*appended.lock().unwrap(), vec![0, 1, 2, 3, 4]); + } + + #[tokio::test] + async fn streams_eagerly_without_waiting_for_late_indices() { + // Submit 0..=2 immediately; index 4 only after waiting. Assert that + // 0/1/2 are released *before* we submit 4 — i.e., the sink doesn't + // buffer to the end. + let appended = Arc::new(Mutex::new(Vec::new())); + let writer = Arc::new(Recorder { + appended: appended.clone(), + }); + let sink = OrderedSink::new(writer, 0); + for i in 0..=2 { + sink.append_at(i, row(i)).await.unwrap(); + } + // Give the drain task a chance to forward 0/1/2. + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + assert_eq!(*appended.lock().unwrap(), vec![0, 1, 2]); + // Submit a row at 4 (gap at 3) — should NOT be released yet. + sink.append_at(4, row(4)).await.unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + assert_eq!(*appended.lock().unwrap(), vec![0, 1, 2]); + // Now fill the gap. + sink.append_at(3, row(3)).await.unwrap(); + sink.close().await.unwrap(); + assert_eq!(*appended.lock().unwrap(), vec![0, 1, 2, 3, 4]); + } + + #[tokio::test] + async fn closing_with_unfilled_gap_returns_error() { + let appended = Arc::new(Mutex::new(Vec::new())); + let writer = Arc::new(Recorder { + appended: appended.clone(), + }); + let sink = OrderedSink::new(writer, 0); + sink.append_at(0, row(0)).await.unwrap(); + sink.append_at(2, row(2)).await.unwrap(); + // Index 1 never submitted. + let err = sink.close().await.unwrap_err(); + assert!( + err.to_string().contains("gap"), + "unexpected error: {}", + err + ); + // 0 went through; 2 was held back. + assert_eq!(*appended.lock().unwrap(), vec![0]); + } + + #[tokio::test] + async fn duplicate_future_index_returns_error() { + let appended = Arc::new(Mutex::new(Vec::new())); + let writer = Arc::new(Recorder { + appended: appended.clone(), + }); + let sink = OrderedSink::new(writer, 0); + // Two submissions at index 5 with the slot still pending (0..=4 never + // arrive). The drain task sees both in pending and reports the + // duplicate; the send itself succeeds because the channel buffered it. + sink.append_at(5, row(5)).await.unwrap(); + sink.append_at(5, row(5)).await.unwrap(); + let err = sink.close().await.unwrap_err(); + assert!( + err.to_string().contains("duplicate"), + "unexpected error: {}", + err + ); + } + + #[tokio::test] + async fn direct_sink_forwards_in_arrival_order_without_buffering() { + // Pass-through variant: index is ignored, rows reach the writer in + // the order they were submitted. Verifies that targets opting out of + // ordering don't accidentally pay the buffering tax. + let appended = Arc::new(Mutex::new(Vec::new())); + let writer = Arc::new(Recorder { + appended: appended.clone(), + }); + let sink: AppendSink = AppendSink::new(writer, 0, false); + // Submit out of order — direct sink does NOT re-order. + sink.append_at(2, row(2)).await.unwrap(); + sink.append_at(0, row(0)).await.unwrap(); + sink.append_at(1, row(1)).await.unwrap(); + sink.close().await.unwrap(); + assert_eq!(*appended.lock().unwrap(), vec![2, 0, 1]); + } + + #[tokio::test] + async fn ordered_sink_via_appendsink_reorders() { + // Sanity that AppendSink dispatches to the ordered path correctly. + let appended = Arc::new(Mutex::new(Vec::new())); + let writer = Arc::new(Recorder { + appended: appended.clone(), + }); + let sink: AppendSink = AppendSink::new(writer, 0, true); + sink.append_at(2, row(2)).await.unwrap(); + sink.append_at(0, row(0)).await.unwrap(); + sink.append_at(1, row(1)).await.unwrap(); + sink.close().await.unwrap(); + assert_eq!(*appended.lock().unwrap(), vec![0, 1, 2]); + } + + #[tokio::test] + async fn abandon_drops_pending_without_gap_error() { + // Two rows submitted with a gap at index 1 — `close` would return a + // gap error, but `abandon` is the explicit "this run is doomed" path + // and must succeed quietly. + let appended = Arc::new(Mutex::new(Vec::new())); + let writer = Arc::new(Recorder { + appended: appended.clone(), + }); + let sink = OrderedSink::new(writer, 0); + sink.append_at(0, row(0)).await.unwrap(); + sink.append_at(2, row(2)).await.unwrap(); + // Give the drain task a chance to forward index 0. + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + // Index 1 never arrives — abandon must not raise the gap error. + sink.abandon().await.unwrap(); + // Whatever was already forwarded stays; pending rows are discarded. + assert_eq!(*appended.lock().unwrap(), vec![0]); + } + + /// Writer that fails its first `append` call. Used to assert that + /// `abandon` surfaces real writer errors instead of masking them as + /// "clean cancellation". + struct FailingWriter; + + impl TargetFile for FailingWriter { + fn get_url(&self) -> String { + "test://failing".to_string() + } + } + + #[async_trait] + impl TargetFileWriter for FailingWriter { + async fn append(&self, _row: ArchiveRow) -> Result<()> { + Err(anyhow!("simulated broker reject")) + } + async fn close(self) -> Result<()> { + Ok(()) + } + } + + #[tokio::test] + async fn abandon_propagates_writer_error_that_surfaced_before_cancel() { + // The underlying writer fails on append; the drain task returns that + // failure. If abandon swallows it as a clean cancellation, callers + // would never learn the broker rejected real messages — exactly the + // failure mode we want to prevent. + let writer = Arc::new(FailingWriter); + let sink = OrderedSink::new(writer, 0); + sink.append_at(0, row(0)).await.unwrap(); + // Give the drain task a chance to consume and fail. + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + let err = sink.abandon().await.unwrap_err(); + assert!( + err.to_string().contains("simulated broker reject"), + "abandon must surface the underlying writer error, got: {}", + err + ); + } + + #[tokio::test] + async fn appendsink_abandon_is_a_noop_for_direct() { + // Pass-through variant has nothing to abandon — call must succeed + // and not affect already-forwarded rows. + let appended = Arc::new(Mutex::new(Vec::new())); + let writer = Arc::new(Recorder { + appended: appended.clone(), + }); + let sink: AppendSink = AppendSink::new(writer, 0, false); + sink.append_at(0, row(0)).await.unwrap(); + sink.abandon().await.unwrap(); + assert_eq!(*appended.lock().unwrap(), vec![0]); + } + + #[tokio::test] + async fn replay_below_cursor_returns_error() { + let appended = Arc::new(Mutex::new(Vec::new())); + let writer = Arc::new(Recorder { + appended: appended.clone(), + }); + let sink = OrderedSink::new(writer, 0); + sink.append_at(0, row(0)).await.unwrap(); + // Give the drain task a chance to forward index 0 and advance cursor. + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + // Now re-submit at index 0 — already past the cursor, so it's a + // logic error from the caller. + sink.append_at(0, row(0)).await.unwrap(); + let err = sink.close().await.unwrap_err(); + assert!( + err.to_string().contains("below cursor"), + "unexpected error: {}", + err + ); + } +} diff --git a/src/archiver/resume.rs b/src/archiver/resume.rs new file mode 100644 index 0000000..a1171a3 --- /dev/null +++ b/src/archiver/resume.rs @@ -0,0 +1,106 @@ +// Copyright 2026 EmeraldPay Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. + +//! Stream-resume strategies. +//! +//! When the `stream` command starts with `--continue`, it asks the target to +//! report which heights/fields are already archived and re-archives the +//! missing tail. The mechanism is target-specific: +//! +//! - File-based backends ([`crate::storage::ScanTarget`]) can list existing +//! files and compute the missing kinds per range. See [`ScanResume`]. +//! - Streaming backends (Pulsar, future Kafka) will eventually grow a +//! broker-side variant that tail-reads each topic; the v1 implementation +//! ships without it. +//! +//! The trait lives in the archiver module rather than alongside the +//! `stream` command because it's an archival operation (it calls back into +//! [`crate::archiver::Archiver::archive`]); the command just dispatches it. + +use std::sync::Arc; + +use anyhow::Result; +use async_trait::async_trait; +use tokio_util::sync::CancellationToken; + +use crate::archiver::archiver::Archiver; +use crate::archiver::datakind::DataOptions; +use crate::archiver::range::{Height, Range}; +use crate::archiver::ArchiveAll; +use crate::blockchain::BlockchainTypes; +use crate::notify::RunMode; +use crate::storage::ScanTarget; + +/// Resume strategy invoked once at stream startup, *before* the first new +/// block is published. Implementations re-archive the tail of the chain so +/// that gaps left by a previous run are filled in before live tailing begins. +#[async_trait] +pub trait StreamResume: Send + Sync { + /// Make the archive complete up to (and including) `up_to`'s parent + /// height, then return. Errors are the caller's to handle — typically + /// callers log and continue so a transient resume failure doesn't block + /// fresh blocks from being archived. + async fn ensure_continued(&self, up_to: Height) -> Result<()>; +} + +/// [`ScanTarget`]-backed [`StreamResume`] implementation that walks the last +/// `continue_blocks` heights of the archive and re-archives anything reported +/// as incomplete. +pub struct ScanResume { + archiver: Archiver, + continue_blocks: u64, + data_options: DataOptions, +} + +impl ScanResume { + pub fn new(archiver: Archiver, continue_blocks: u64, data_options: DataOptions) -> Self { + Self { archiver, continue_blocks, data_options } + } + + /// Convenience constructor that hands back the boxed trait object the + /// command layer wants to store. Saves callers from importing + /// `Arc` and the trait at once. + pub fn boxed( + archiver: Archiver, + continue_blocks: u64, + data_options: DataOptions, + ) -> Arc + where + B: 'static, + TS: 'static, + { + Arc::new(Self::new(archiver, continue_blocks, data_options)) + } +} + +#[async_trait] +impl StreamResume for ScanResume +where + B: BlockchainTypes + 'static, + TS: ScanTarget + 'static, +{ + async fn ensure_continued(&self, height: Height) -> Result<()> { + let range = Range::up_to(self.continue_blocks, &Range::Single(height)); + let options = self.data_options.clone(); + let missing = self + .archiver + .target + .find_incomplete_tables(range, &options) + .await?; + // Resume runs against settled blocks (we're back-filling a tail the + // previous process didn't finish), so there's no re-org signal — use + // a never-cancelled token. + let cancel = CancellationToken::new(); + for (range, kinds) in missing { + let range_opts = options.clone().only_include(&kinds); + for height in range.iter().collect::>() { + self.archiver + .archive(Height::from(height), RunMode::Stream, None, &range_opts, &cancel) + .await?; + } + } + Ok(()) + } +} diff --git a/src/archiver/table.rs b/src/archiver/table.rs index 90cb6a9..6a0bc35 100644 --- a/src/archiver/table.rs +++ b/src/archiver/table.rs @@ -3,8 +3,10 @@ use anyhow::anyhow; use chrono::Utc; use tokio::sync::Semaphore; use tokio::task::JoinSet; +use tokio_util::sync::CancellationToken; use crate::archiver::archiver::Archiver; -use crate::archiver::BlockTransactions; +use crate::archiver::order::AppendSink; +use crate::archiver::{BlockTransactions, ProcessOutcome}; use crate::blockchain::{BlockchainData, BlockchainTypes}; use crate::archiver::datakind::{DataKind, DataOptions}; use crate::notify::Notification; @@ -14,10 +16,20 @@ use crate::storage::{TargetFile, TargetFileWriter, WriteTarget}; impl Archiver { - pub async fn process_traces(&self, range: Range, notification: Notification, blocks: &BlockTransactions, options: &DataOptions) -> anyhow::Result<()> { + pub async fn process_traces( + &self, + range: Range, + notification: Notification, + blocks: &BlockTransactions, + options: &DataOptions, + cancel: &CancellationToken, + ) -> anyhow::Result> { let shutdown = global::get_shutdown(); if shutdown.is_signalled() { - return Ok(()); + return Ok(ProcessOutcome::Completed { + value: (), + notification: None, + }); } let dry_run = global::is_dry_run(); let file = self.target.create(DataKind::TransactionTraces, &range, options.overwrite) @@ -25,38 +37,61 @@ impl Archiver { .map_err(|e| anyhow!("Unable to create file: {}", e))?; if file.is_none() { tracing::debug!(range = %range, "Skipping existing file"); - return Ok(()); + return Ok(ProcessOutcome::Completed { + value: (), + notification: None, + }); } let file = file.unwrap(); let options = options.trace.as_ref().unwrap(); let file_url = file.get_url(); let file = Arc::new(file); + // Order traces by a flat `(block_position, tx_index)` ordinal so block + // N's traces are all published before block N+1's, and within a block + // traces follow tx_index order. The sink only buffers/reorders when + // the target asks for ordering — file backends get a pass-through. + // See `process_txes` for the same pattern. + let sink = Arc::new(AppendSink::new( + file.clone(), + 0, + self.target.needs_ordering(), + )); let mut jobs = JoinSet::new(); let semaphore = Arc::new(Semaphore::new(global::get_threads().trace)); let options = Arc::new(options.clone()); + let mut flat_index: u64 = 0; for (block, txes) in blocks.iter() { let block = Arc::new(block.clone()); for tx_index in 0..txes.len() { let block = block.clone(); let provider = self.data_provider.clone(); let options = options.clone(); - let file = file.clone(); + let sink = sink.clone(); let shutdown = shutdown.clone(); let semaphore = semaphore.clone(); + let order_idx = flat_index; + let cancel = cancel.clone(); + flat_index += 1; jobs.spawn(async move { if shutdown.is_signalled() { return Ok(()); } let _permit = semaphore.acquire().await.unwrap(); - let data = provider.fetch_traces(&block, tx_index, &options).await?; - if !dry_run { - file.append(data).await?; + let work = async { + let data = provider.fetch_traces(&block, tx_index, &options).await?; + if !dry_run { + sink.append_at(order_idx, data).await?; + } + crate::progress::on_record(); + crate::metrics::add_items(&DataKind::TransactionTraces, crate::metrics::Direction::Write, 1); + Ok::<_, anyhow::Error>(()) + }; + tokio::select! { + _ = cancel.cancelled() => Ok(()), + r = work => r, } - crate::progress::on_record(); - crate::metrics::add_items(&DataKind::TransactionTraces, crate::metrics::Direction::Write, 1); - Ok::<_, anyhow::Error>(()) }); } } @@ -65,28 +100,54 @@ impl Archiver { res.map_err(|e| anyhow!("Task failed: {}", e))??; } + // See `process_blocks` for the rationale: on cancel we abandon the + // ordering layer and let the writer Drop clean up (delete the + // partial file on disk / abandon the multipart upload on S3 / no-op + // for streaming brokers whose messages have already been published). + if cancel.is_cancelled() { + if !dry_run { + let sink = Arc::into_inner(sink) + .ok_or_else(|| anyhow!("AppendSink still referenced after all tasks completed"))?; + sink.abandon().await?; + drop(file); + } + return Ok(ProcessOutcome::Cancelled); + } + if !dry_run { + let sink = Arc::into_inner(sink) + .ok_or_else(|| anyhow!("AppendSink still referenced after all tasks completed"))?; + sink.close().await?; let file = Arc::into_inner(file) .ok_or_else(|| anyhow!("File writer still referenced after all tasks completed"))?; let _ = file.close().await?; } - let notification_tx = Notification { + let notification = Notification { file_type: DataKind::TransactionTraces, location: file_url, ts: Utc::now(), - ..notification }; - if !dry_run { - let _ = self.notifications.send(notification_tx).await; - } - Ok(()) + Ok(ProcessOutcome::Completed { + value: (), + notification: Some(notification), + }) } - pub async fn process_txes(&self, range: Range, notification: Notification, blocks: &BlockTransactions, options: &DataOptions) -> anyhow::Result<()> { + pub async fn process_txes( + &self, + range: Range, + notification: Notification, + blocks: &BlockTransactions, + options: &DataOptions, + cancel: &CancellationToken, + ) -> anyhow::Result> { let shutdown = global::get_shutdown(); if shutdown.is_signalled() { - return Ok(()); + return Ok(ProcessOutcome::Completed { + value: (), + notification: None, + }); } let dry_run = global::is_dry_run(); let file = self.target.create(DataKind::Transactions, &range, options.overwrite) @@ -94,35 +155,61 @@ impl Archiver { .map_err(|e| anyhow!("Unable to create file: {}", e))?; if file.is_none() { tracing::debug!(range = %range, "Skipping existing file"); - return Ok(()); + return Ok(ProcessOutcome::Completed { + value: (), + notification: None, + }); } let file = file.unwrap(); let file_url = file.get_url(); let file = Arc::new(file); + // Tx ordering: a flat `(block_position, tx_index)` ordinal across the + // whole range. `blocks` is already sorted by height by + // `process_blocks`, so walking it linearly yields chain order; + // numbering tasks 0, 1, 2, … as we enumerate guarantees that block N's + // txes (in their natural tx_index order) precede block N+1's, even + // when fetches finish out of order. The sink only buffers/reorders + // when the target asks for ordering (streaming backends); file + // backends get a pass-through. + let sink = Arc::new(AppendSink::new( + file.clone(), + 0, + self.target.needs_ordering(), + )); let mut jobs = JoinSet::new(); let semaphore = Arc::new(Semaphore::new(global::get_threads().tx)); + let mut flat_index: u64 = 0; for (block, txes) in blocks.iter() { let block = Arc::new(block.clone()); for tx_index in 0..txes.len() { let block = block.clone(); let provider = self.data_provider.clone(); - let file = file.clone(); + let sink = sink.clone(); let shutdown = shutdown.clone(); let semaphore = semaphore.clone(); + let order_idx = flat_index; + let cancel = cancel.clone(); + flat_index += 1; jobs.spawn(async move { if shutdown.is_signalled() { return Ok(()); } let _permit = semaphore.acquire().await.unwrap(); - let data = provider.fetch_tx(&block, tx_index).await?; - if !dry_run { - file.append(data).await?; + let work = async { + let data = provider.fetch_tx(&block, tx_index).await?; + if !dry_run { + sink.append_at(order_idx, data).await?; + } + crate::progress::on_record(); + crate::metrics::add_items(&DataKind::Transactions, crate::metrics::Direction::Write, 1); + Ok::<_, anyhow::Error>(()) + }; + tokio::select! { + _ = cancel.cancelled() => Ok(()), + r = work => r, } - crate::progress::on_record(); - crate::metrics::add_items(&DataKind::Transactions, crate::metrics::Direction::Write, 1); - Ok::<_, anyhow::Error>(()) }); } } @@ -131,21 +218,37 @@ impl Archiver { res.map_err(|e| anyhow!("Task failed: {}", e))??; } + // See `process_blocks` for the rationale: on cancel we abandon the + // ordering layer and let the writer Drop clean up the partial + // artifact. The notification is built but only published by + // `archive()` once the entire run is known to be uncancelled. + if cancel.is_cancelled() { + if !dry_run { + let sink = Arc::into_inner(sink) + .ok_or_else(|| anyhow!("AppendSink still referenced after all tasks completed"))?; + sink.abandon().await?; + drop(file); + } + return Ok(ProcessOutcome::Cancelled); + } + if !dry_run { + let sink = Arc::into_inner(sink) + .ok_or_else(|| anyhow!("AppendSink still referenced after all tasks completed"))?; + sink.close().await?; let file = Arc::into_inner(file) .ok_or_else(|| anyhow!("File writer still referenced after all tasks completed"))?; let _ = file.close().await?; } - let notification_tx = Notification { + let notification = Notification { file_type: DataKind::Transactions, location: file_url, ts: Utc::now(), - ..notification }; - if !dry_run { - let _ = self.notifications.send(notification_tx).await; - } - Ok(()) + Ok(ProcessOutcome::Completed { + value: (), + notification: Some(notification), + }) } } diff --git a/src/args.rs b/src/args.rs index e92d2d2..1839d99 100644 --- a/src/args.rs +++ b/src/args.rs @@ -60,6 +60,9 @@ pub struct Args { #[command(flatten)] pub aws: Option, + #[command(flatten)] + pub stream: Option, + /// Target directory #[arg(long = "dir", short)] pub dir: Option, @@ -99,10 +102,25 @@ pub struct Args { pub fix_clean: bool, /// - /// Compression algorithm to use when writing new Avro files. Default is `zstd`. + /// Compression algorithm to use for new output. For `--format=avro` it's + /// the Avro file codec; for the Pulsar streaming target it's the producer + /// compression applied to every message. Default is `zstd`. #[arg(long = "compression")] pub compression: Option, + /// Retry policy for transient blockchain fetch failures. + /// + /// - `bounded` — give up after a fixed number of attempts (current + /// behaviour for file targets). + /// - `forever` — keep retrying indefinitely with exponential backoff. + /// Required for ordered streaming targets where a missed record breaks + /// the topic-order contract. + /// + /// Defaults: `forever` for streaming-ordered targets (e.g. Pulsar), + /// `bounded` otherwise. + #[arg(long = "retry")] + pub retry: Option, + /// /// [Stream Command] Follow mode for new blocks: `latest` - follow the latest blocks (default); `finalized` - follow only finalized blocks #[arg(long = "follow", default_value = "latest")] @@ -137,6 +155,7 @@ impl Default for Args { connection: Connection::default(), notify: None, aws: None, + stream: None, dir: None, continue_last: false, tail: None, @@ -146,6 +165,7 @@ impl Default for Args { fields_trace: Some("calls,stateDiff".to_string()), fix_clean: false, compression: None, + retry: None, follow: Follow::Latest, format: Format::Avro, metrics: None, @@ -259,6 +279,53 @@ pub struct Aws { pub trust_tls: bool, } +/// Streaming-target options. Picked up only by the `stream` command. +/// +/// The broker is identified by the URL scheme on `--stream.url`: `pulsar://` +/// selects Apache Pulsar. When `--stream.url` is set the archive runs against +/// a topic-per-field broker target instead of a file backend, so `--dir`, +/// `--auth.aws.*`, and `--format` are ignored. +#[derive(Parser, Debug, Clone)] +pub struct Stream { + /// Publish stream data to a broker at the given URL. The scheme selects + /// the backend: + /// + /// - `pulsar://HOST:PORT` — Apache Pulsar. + /// + /// Selecting a streaming target restricts the run to the `stream` command — + /// `archive`, `fix`, `verify`, and `compact` are rejected at startup + /// because topics are append-only. + #[arg(long = "stream.url", required = false, alias = "stream-url")] + pub stream_url: Option, + + /// Prefix used to build the per-field topic names. Each field is published + /// to `-` (e.g. `-blocks`, `-tx-json`). + /// For Pulsar, include the full topic path up to the prefix, e.g. + /// `persistent://public/default/archive-eth`. + #[arg(long = "stream.topics", required = false, alias = "stream-topics")] + pub stream_topics: Option, +} + +impl Default for Stream { + fn default() -> Self { + Self { + stream_url: None, + stream_topics: None, + } + } +} + +impl Stream { + /// True when the args carry a Pulsar streaming target (URL scheme + /// `pulsar://`). Other schemes will route to other backends in the future. + pub fn is_pulsar(&self) -> bool { + self.stream_url + .as_deref() + .map(|u| u.starts_with("pulsar://")) + .unwrap_or(false) + } +} + impl Default for Aws { fn default() -> Self { Self { @@ -278,6 +345,21 @@ pub enum Compression { Zstd, } +/// CLI-facing retry mode (see [`Args::retry`]). +/// +/// The runtime translates this into a [`crate::global::RetryPolicy`] at +/// startup; the indirection lets us extend the policy with options (e.g. a +/// configurable max-attempts) without touching the CLI surface. +#[derive(clap::ValueEnum, Debug, Clone, Copy, PartialEq, Eq)] +pub enum RetryMode { + /// Give up after a fixed number of attempts. Default for file targets, + /// where a failed fetch can be repaired later by the `fix` command. + Bounded, + /// Keep retrying indefinitely. Default for streaming-ordered targets + /// (Pulsar) where a gap breaks the topic-order contract permanently. + Forever, +} + /// Output format selected via `--format`. #[derive(clap::ValueEnum, Debug, Clone, Copy, PartialEq, Eq)] pub enum Format { diff --git a/src/blockchain/bitcoin.rs b/src/blockchain/bitcoin.rs index 55c9430..6bea5d4 100644 --- a/src/blockchain/bitcoin.rs +++ b/src/blockchain/bitcoin.rs @@ -155,6 +155,7 @@ impl BlockchainData for BitcoinData { parent_id: Some(format!("{:x}", &parsed_block.previous_block_hash)), tx_index: None, tx_id: None, + tx_count: Some(parsed_block.transactions.len() as u64), fields: vec![Field::BlockJson(raw_block)], }; @@ -178,9 +179,10 @@ impl BlockchainData for BitcoinData { height: block.height, block_id: format!("{:x}", &block.hash), timestamp: block_timestamp(block.time), - parent_id: None, + parent_id: Some(format!("{:x}", &block.previous_block_hash)), tx_index: Some(index as u64), tx_id: Some(format!("{:x}", tx_hash)), + tx_count: Some(block.transactions.len() as u64), fields: vec![ Field::TxJson(tx?), Field::TxRaw(tx_raw?), diff --git a/src/blockchain/block_seq.rs b/src/blockchain/block_seq.rs index 9c725af..b0b7ac9 100644 --- a/src/blockchain/block_seq.rs +++ b/src/blockchain/block_seq.rs @@ -149,6 +149,14 @@ impl BlockSequence { Some((top.height, &top.blocks.first().unwrap().current)) } + /// + /// True when no blocks have been appended yet. Used by walk-back logic + /// that needs to stop on the very first head event (no ancestors exist + /// to reconnect to). + pub fn is_empty(&self) -> bool { + self.current.is_empty() + } + /// /// Finds the index of the given height in the current heights, if any. fn get_index(&self, height: u64) -> Option { diff --git a/src/blockchain/ethereum.rs b/src/blockchain/ethereum.rs index 8ff5c6f..3f42240 100644 --- a/src/blockchain/ethereum.rs +++ b/src/blockchain/ethereum.rs @@ -1,5 +1,4 @@ use std::sync::{Arc}; -use std::time::Duration; use async_trait::async_trait; use crate::errors::{BlockchainError}; use crate::blockchain::connection::{Blockchain}; @@ -9,10 +8,9 @@ use alloy::{ rpc::types::{Transaction as TransactionJson, Block as BlockJson, Block, TransactionTrait} }; use alloy::network::TransactionResponse; -use crate::blockchain::{BlockDetails, BlockReference, BlockchainData, BlockchainTypes, EthereumType, JsonString}; +use crate::blockchain::{BlockDetails, BlockHeaderInfo, BlockReference, BlockchainData, BlockchainTypes, EthereumType, JsonString}; use anyhow::{Result, anyhow}; use tokio_retry2::{Retry, RetryError}; -use tokio_retry2::strategy::{jitter, ExponentialFactorBackoff}; use crate::archiver::datakind::{DataKind, TraceOptions}; use crate::blockchain::next_block::{NextBlock, NextFinalizedBlock}; use crate::record::{ArchiveRow, BlockchainType as ArchiveBlockchainType, Field}; @@ -23,10 +21,14 @@ pub struct EthereumData { blockchain_id: String, } -fn create_exp_retry() -> ExponentialFactorBackoff { - ExponentialFactorBackoff::from_millis(100, 1.75) - .max_delay(Duration::from_secs(2)) -} +/// Default cap on the time between retry attempts. Higher than the typical +/// 95th-percentile RPC latency so a degraded node has space to recover, but +/// low enough that a transient blip doesn't stall a block for noticeably long. +const RETRY_MAX_DELAY_FAST_SECS: u64 = 2; + +/// Cap used by the trace/state-diff helpers — these RPCs are heavier (full +/// `debug_traceTransaction` runs), so a longer backoff is appropriate. +const RETRY_MAX_DELAY_TRACE_SECS: u64 = 5; impl EthereumData { @@ -73,11 +75,19 @@ impl EthereumData { Ok(data) } - async fn get_tx(&self, hash: &TxHash) -> Result> { - tracing::debug!(tx_hash = %format!("0x{:x}", hash), "Get transaction"); - let params = format!("[\"0x{:x}\"]", hash).as_bytes().to_vec(); - let data = self.blockchain.native_call("eth_getTransactionByHash", params).await?; - Ok(data) + async fn get_tx_at(&self, block: &BlockHash, i: usize) -> Result> { + tracing::debug!(block_hash = %format!("0x{:x}", block), tx_index = %i, "Get transaction"); + let retry_strategy = crate::global::retry_strategy(RETRY_MAX_DELAY_FAST_SECS); + Retry::spawn(retry_strategy, async || { + let params = format!("[\"0x{:x}\", \"{:#01x}\"]", block, i).as_bytes().to_vec(); + self.blockchain.native_call("eth_getTransactionByBlockHashAndIndex", params).await + .and_then(|value| if value == b"null" { + Err(BlockchainError::InvalidResponse) + } else { + Ok(value) + }) + .map_err(|e| RetryError::transient(e)) + }).await.map_err(|e| anyhow!("Failed to get transaction at block 0x{:x} index {}: {}", block, i, e)) } async fn get_tx_receipt(&self, hash: &TxHash) -> Result> { @@ -102,25 +112,8 @@ impl EthereumData { hex::decode(data_as_hex).map_err(|_| anyhow!("Invalid hex")) } - async fn get_tx_expected(&self, hash: &TxHash) -> Result> { - let retry_strategy = create_exp_retry() - .map(jitter) - .take(10); - Retry::spawn(retry_strategy, async || { - self.get_tx(hash).await - .and_then(|value| if value == b"null" { - Err(anyhow!("Transaction not found: 0x{:x}", hash)) - } else { - Ok(value) - }) - .map_err(|e| RetryError::transient(e)) - }).await - } - async fn get_tx_receipt_expected(&self, hash: &TxHash) -> Result> { - let retry_strategy = create_exp_retry() - .map(jitter) - .take(10); + let retry_strategy = crate::global::retry_strategy(RETRY_MAX_DELAY_FAST_SECS); Retry::spawn(retry_strategy, async || { self.get_tx_receipt(hash).await .and_then(|value| if value == b"null" { @@ -133,9 +126,7 @@ impl EthereumData { } async fn get_tx_raw_expected(&self, hash: &TxHash) -> Result> { - let retry_strategy = create_exp_retry() - .map(jitter) - .take(10); + let retry_strategy = crate::global::retry_strategy(RETRY_MAX_DELAY_FAST_SECS); Retry::spawn(retry_strategy, async || { self.get_tx_raw(hash).await .and_then(|value| if value.is_empty() { @@ -157,10 +148,7 @@ impl EthereumData { let blockchain = self.blockchain.clone(); let hash = hash.clone(); - let retry_strategy = create_exp_retry() - .max_delay(Duration::from_secs(5)) - .map(jitter) - .take(10); + let retry_strategy = crate::global::retry_strategy(RETRY_MAX_DELAY_TRACE_SECS); Retry::spawn(retry_strategy, async || { blockchain.native_call("debug_traceTransaction", params.clone()).await .map_err(|e| anyhow!("Failed to get transaction trace: {}", e)) @@ -186,10 +174,7 @@ impl EthereumData { let blockchain = self.blockchain.clone(); let hash = hash.clone(); - let retry_strategy = create_exp_retry() - .max_delay(Duration::from_secs(5)) - .map(jitter) - .take(10); + let retry_strategy = crate::global::retry_strategy(RETRY_MAX_DELAY_TRACE_SECS); Retry::spawn(retry_strategy, async || { blockchain.native_call("debug_traceTransaction", params.clone()).await .map_err(|e| anyhow!("Failed to get transaction trace: {}", e)) @@ -215,9 +200,10 @@ fn tx_row(kind: DataKind, blockchain_id: String, block: &Block, index: u height: block.header.number, block_id: format!("0x{:x}", &block.header.hash), timestamp: block_timestamp(block.header.timestamp), - parent_id: None, + parent_id: Some(format!("0x{:x}", &block.header.parent_hash)), tx_index: Some(index as u64), tx_id: Some(format!("0x{:x}", tx_hash)), + tx_count: Some(block.transactions.len() as u64), fields: Vec::new(), } } @@ -261,6 +247,7 @@ impl BlockchainData for EthereumData { parent_id: Some(format!("0x{:x}", &parsed_block.header.parent_hash)), tx_index: None, tx_id: None, + tx_count: Some(parsed_block.transactions.len() as u64), fields, }; @@ -273,20 +260,42 @@ impl BlockchainData for EthereumData { Ok((row, parsed_block, transactions)) } + /// Cheap header-only path: pulls one `eth_getBlockBy{Hash,Number}` and + /// projects just the three fields the re-org follower needs. Skips uncle + /// RPCs and full row construction. + async fn fetch_block_link( + &self, + reference: &BlockReference, + ) -> Result { + let raw = match reference { + BlockReference::Hash(hash) => self.get_block(hash).await?, + BlockReference::Height(h) => self.get_block_at(h.height).await?, + }; + let parsed = serde_json::from_slice::>(raw.as_slice()) + .map_err(|_| BlockchainError::InvalidResponse)?; + Ok(BlockHeaderInfo { + height: parsed.header.number, + hash: format!("0x{:x}", parsed.header.hash), + parent: format!("0x{:x}", parsed.header.parent_hash), + }) + } + async fn fetch_tx(&self, block: &Block, index: usize) -> Result { + let block_hash = block.header.hash.clone(); let tx_hash = block.transactions.as_transactions().map(|txes| txes[index]) .ok_or_else(|| anyhow!("Transaction not found"))?; + // Fetch all transaction data in parallel let (tx_json_bytes, tx_raw, tx_receipt) = tokio::join!( - self.get_tx_expected(&tx_hash), + self.get_tx_at(&block_hash, index), self.get_tx_raw_expected(&tx_hash), self.get_tx_receipt_expected(&tx_hash), ); let tx_json_bytes = tx_json_bytes?; let parsed_tx = serde_json::from_slice::(tx_json_bytes.as_slice()) - .map_err(|e| anyhow!("Invalid Transaction JSON: {}", e))?; + .map_err(|e| anyhow!("Invalid Transaction JSON: {} from {}", e, String::from_utf8_lossy(tx_json_bytes.as_slice()).to_string()))?; let mut row = tx_row(DataKind::Transactions, self.blockchain_id(), block, index, &tx_hash); row.fields.push(Field::TxJson(tx_json_bytes)); diff --git a/src/blockchain/mock.rs b/src/blockchain/mock.rs index 3ccd876..7aa7920 100644 --- a/src/blockchain/mock.rs +++ b/src/blockchain/mock.rs @@ -13,6 +13,9 @@ use crate::record::{ArchiveRow, BlockchainType as ArchiveBlockchainType, Field}; pub struct MockType {} impl BlockchainTypes for MockType { + // Mock uses Ethereum's discriminator — Mock-specific tests assert against + // ArchiveBlockchainType::Ethereum elsewhere, so this keeps them consistent. + const BLOCKCHAIN_TYPE: ArchiveBlockchainType = ArchiveBlockchainType::Ethereum; type BlockHash = String; type TxId = String; @@ -124,9 +127,10 @@ impl BlockchainData for MockData { height: block.height, block_id: block.hash.clone(), timestamp: Utc.timestamp_millis_opt(1).unwrap(), - parent_id: Some(String::new()), + parent_id: Some(block.parent.clone()), tx_index: None, tx_id: None, + tx_count: Some(block.transactions.len() as u64), fields: vec![Field::BlockJson(serde_json::to_vec(&block).unwrap())], }; @@ -153,6 +157,7 @@ impl BlockchainData for MockData { parent_id: None, tx_index: Some(index as u64), tx_id: Some(tx.hash.clone()), + tx_count: Some(block.transactions.len() as u64), fields: vec![ Field::TxJson(json.clone()), Field::TxRaw(json), @@ -179,6 +184,7 @@ impl BlockchainData for MockData { parent_id: None, tx_index: Some(index as u64), tx_id: Some(tx_hash.clone()), + tx_count: Some(block.transactions.len() as u64), fields: Vec::new(), }; diff --git a/src/blockchain/mod.rs b/src/blockchain/mod.rs index 6557e10..bdf0d7d 100644 --- a/src/blockchain/mod.rs +++ b/src/blockchain/mod.rs @@ -22,7 +22,7 @@ use crate::{ archiver::{ datakind::TraceOptions, }, - record::ArchiveRow, + record::{ArchiveRow, BlockchainType}, }; use crate::archiver::range::Height; @@ -30,6 +30,12 @@ use crate::archiver::range::Height; /// Defined the data types for a blockchain pub trait BlockchainTypes: Send + Sync + Sized { + /// + /// Runtime discriminator for the blockchain family. Lets generic code + /// (e.g. the Pulsar topic-creation path) branch on Bitcoin vs Ethereum + /// without needing a `match` on the type-erased provider. + const BLOCKCHAIN_TYPE: BlockchainType; + /// /// Type of the Block Hash / Block Identifier type BlockHash: FromStr + PartialEq + Hash + Eq + Send + Sync + Debug + Clone + 'static; @@ -52,6 +58,8 @@ pub trait BlockchainTypes: Send + Sync + Sized { pub struct EthereumType {} impl BlockchainTypes for EthereumType { + const BLOCKCHAIN_TYPE: BlockchainType = BlockchainType::Ethereum; + type BlockHash = alloy::primitives::BlockHash; type TxId = alloy::primitives::TxHash; type BlockParsed = alloy::rpc::types::Block; @@ -64,6 +72,8 @@ impl BlockchainTypes for EthereumType { } pub struct BitcoinType {} impl BlockchainTypes for BitcoinType { + const BLOCKCHAIN_TYPE: BlockchainType = BlockchainType::Bitcoin; + type BlockHash = bitcoin::BlockHash; type TxId = bitcoin::TxHash; type BlockParsed = bitcoin::BitcoinBlock; @@ -90,6 +100,27 @@ pub trait BlockchainData: Send + Sync { /// the parsed block (used to enumerate transactions) and the list of transaction ids. async fn fetch_block(&self, height: &BlockReference) -> Result<(ArchiveRow, T::BlockParsed, Vec)>; + /// + /// Lightweight header-only fetch returning the block's `(height, hash, parent)` + /// linkage. Used by the re-org-aware live follower to walk parent hashes + /// without paying for the full `fetch_block` (which also fetches uncle + /// JSON and builds an [`ArchiveRow`]). + /// + /// The default implementation just calls `fetch_block` and projects the + /// linkage fields; concrete implementations may override with a cheaper + /// path that skips uncle / row construction. + async fn fetch_block_link( + &self, + reference: &BlockReference, + ) -> Result { + let (row, _parsed, _txes) = self.fetch_block(reference).await?; + Ok(BlockHeaderInfo { + height: row.height, + hash: row.block_id, + parent: row.parent_id.unwrap_or_default(), + }) + } + /// /// Get the details for the transaction. async fn fetch_tx(&self, block: &T::BlockParsed, index: usize) -> Result; @@ -170,6 +201,20 @@ pub trait BlockDetails where T: BlockchainTypes{ fn parent(&self) -> T::BlockHash; } +/// +/// Lightweight block-header linkage returned by +/// [`BlockchainData::fetch_block_link`]. The string fields use the same +/// formatting convention as [`crate::record::ArchiveRow::block_id`] and +/// [`crate::record::ArchiveRow::parent_id`] (chain-specific; e.g. `0x…` for +/// Ethereum) so the values round-trip through the +/// `From for BlockReference` impl. +#[derive(Debug, Clone)] +pub struct BlockHeaderInfo { + pub height: u64, + pub hash: String, + pub parent: String, +} + pub struct JsonString(pub String); impl Into for JsonString { diff --git a/src/blockchain/next_block.rs b/src/blockchain/next_block.rs index 07f7258..cf89f37 100644 --- a/src/blockchain/next_block.rs +++ b/src/blockchain/next_block.rs @@ -1,29 +1,303 @@ +use std::collections::BTreeMap; +use std::collections::btree_map::Entry; +use std::str::FromStr; use std::sync::Arc; use async_trait::async_trait; use tokio::sync::mpsc::Receiver; +use tokio_util::sync::CancellationToken; use crate::archiver::range::Height; use crate::blockchain::block_seq::BlockSequence; -use crate::blockchain::{BlockchainTypes, EthereumType}; +use crate::blockchain::{BlockHeaderInfo, BlockReference, BlockchainData, BlockchainTypes, EthereumType}; use crate::blockchain::connection::{Blockchain}; use crate::errors::BlockchainError; +/// One unit of work emitted by a [`NextBlock`] pump: which block to archive, +/// plus a token the pump can fire to tell the archiver "abandon this one, +/// the chain moved on". +/// +/// Pumps that operate on settled blocks ([`NextFinalizedBlock`], historical +/// archive) emit jobs with a fresh, never-fired [`CancellationToken`] — the +/// token field stays uniform across all pump kinds so the archiver loop +/// doesn't have to branch on the source. Only [`ReorgAwareFollower`] +/// actually fires tokens, when a same-height re-org or deep re-org +/// invalidates a previously-emitted height. +#[derive(Clone, Debug)] +pub struct BlockJob { + pub height: Height, + pub cancel: CancellationToken, +} + +impl BlockJob { + /// Convenience for non-cancelling pumps and tests: wrap a height with a + /// fresh token that will never fire. + pub fn untracked(height: Height) -> Self { + Self { + height, + cancel: CancellationToken::new(), + } + } +} + /// /// Provides next blocks to archive (basically just for Stream archiving mode) #[async_trait] pub trait NextBlock: Send + Sync { - async fn next_blocks(&self) -> Result, BlockchainError>; + async fn next_blocks(&self) -> Result, BlockchainError>; } /// -/// A default implementation that just subscribes to new blocks from the blockchain ("head" subscription) -/// Note that with Head subscription a block may be reorganized later, i.e., a block could be replaced +/// A default implementation that just subscribes to new blocks from the blockchain ("head" subscription). +/// Note that with a Head subscription a block may be reorganized later, i.e., a block could be replaced; +/// this pump does not detect that — every emission carries an untracked token. For re-org awareness, +/// wrap this with [`ReorgAwareFollower`]. #[async_trait] impl NextBlock for Arc { - async fn next_blocks(&self) -> Result, BlockchainError> { - self.subscribe_blocks().await + async fn next_blocks(&self) -> Result, BlockchainError> { + let mut heights = self.subscribe_blocks().await?; + let (tx, rx) = tokio::sync::mpsc::channel(8); + tokio::spawn(async move { + while let Some(h) = heights.recv().await { + if tx.send(BlockJob::untracked(h)).await.is_err() { + break; + } + } + }); + Ok(rx) } } +/// +/// Re-org aware live follower used by `stream --follow=latest`. +/// +/// Wraps a raw head subscription with a [`BlockSequence`] that validates +/// every incoming head event against its parent chain. When the new head's +/// parent isn't already in the sequence, the follower walks backwards by +/// hash (via [`BlockchainData::fetch_block_link`]) until it reconnects to a +/// known ancestor, and then emits the newly-linked heights in chain order +/// (oldest first). +/// +/// This catches two cases the dumb follower misses: +/// +/// - **Same-height re-org**: a new block arrives at a height we already +/// emitted, with a different hash. The walk-back terminates immediately +/// (its parent is the same as the old block's parent — already in the +/// sequence), and we emit the replacement block so the archiver re-publishes +/// it. +/// - **Deep re-org**: the new head's parent doesn't match anything we have. +/// The walk-back fetches ancestors one by one until it reconnects, and the +/// whole affected suffix gets re-emitted so the archiver overwrites the +/// stale heights. +/// +/// Per-height [`CancellationToken`]s are tracked alongside the live block +/// sequence: when a height is re-emitted with a different hash the prior +/// token is cancelled before the replacement [`BlockJob`] goes out, so the +/// archiver task that's still fetching the doomed block can drop its +/// in-flight RPCs and abandon any partially-written ordered sink. +/// +/// Duplicate head events for blocks already in the sequence are silently +/// dropped (the head subscription can re-emit the current tip on +/// reconnections), keeping the live stream idempotent. +pub struct ReorgAwareFollower { + blockchain: Arc, + data_provider: Arc, + /// Number of recent heights to remember when checking parent linkage. + /// 64 covers any realistic re-org depth on production chains + /// (Ethereum finality kicks in at ~64 slots). + history: usize, +} + +impl ReorgAwareFollower { + pub fn new(blockchain: Arc, data_provider: Arc) -> Self { + Self { + blockchain, + data_provider, + history: 64, + } + } +} + +#[async_trait] +impl NextBlock for ReorgAwareFollower { + async fn next_blocks(&self) -> Result, BlockchainError> { + let mut head = self.blockchain.subscribe_blocks().await?; + let (tx, rx) = tokio::sync::mpsc::channel(8); + let data_provider = self.data_provider.clone(); + let history = self.history; + tokio::spawn(async move { + let mut seq = BlockSequence::::new(history); + // Per-height token bookkeeping. Bounded to `history` entries; oldest + // evicted on overflow. Storing the hash alongside the token lets us + // distinguish "same block, retry" from "different block, re-org". + let mut live_tokens: BTreeMap = BTreeMap::new(); + while let Some(head_evt) = head.recv().await { + if let Err(e) = handle_head_event::( + &mut seq, + &mut live_tokens, + history, + &data_provider, + head_evt, + &tx, + ) + .await + { + tracing::warn!("Re-org follower error on head event: {:?}", e); + // Transient errors shouldn't kill the follower — the next + // head event will re-validate the chain on its own. + } + } + tracing::info!("Head subscription ended; re-org follower exiting"); + }); + Ok(rx) + } +} + +/// Process one head event: validate parent linkage, walk back if needed, +/// emit the newly-linked heights in chain order — cancelling any prior +/// token at a height that's being replaced. +/// +/// The walk-back is *atomic with respect to the sequence*: parents are +/// fetched into a local buffer and the sequence is only mutated after the +/// chain has been fully linked. A mid-walk `fetch_block_link` failure +/// returns Err with both `seq` and `live_tokens` unchanged, so the next +/// head event can re-validate from scratch instead of inheriting a +/// partial state. +async fn handle_head_event( + seq: &mut BlockSequence, + live_tokens: &mut BTreeMap, + history: usize, + data_provider: &Arc, + head_evt: Height, + tx: &tokio::sync::mpsc::Sender, +) -> anyhow::Result<()> { + // First fetch the head's linkage to learn its parent hash. The head + // subscription only gives us `(height, hash)`; the parent_hash comes from + // the block header itself. + let head_ref: BlockReference = head_evt.clone().into(); + let head_link = data_provider.fetch_block_link(&head_ref).await?; + + // Idempotency: skip only when the latest *live* hash at this height + // matches the incoming one — i.e., this is the broker re-emitting a + // block we already acted on. A different hash at the same height — + // including a revert back to a previously-canonical block — must NOT + // short-circuit; `seq.get_block` is unsafe for this check because + // `AtHeight.blocks` accumulates and matches any prior emission at this + // height, silently dropping an A→B→A reversion. + if live_tokens + .get(&head_link.height) + .map(|(existing_hash, _)| existing_hash == &head_link.hash) + .unwrap_or(false) + { + tracing::trace!( + "Head event already seen: height={} hash={}", + head_link.height, + head_link.hash + ); + return Ok(()); + } + + // Walk parents into a local buffer without touching `seq`. The walk + // stops when: + // - the parent is already in `seq` at `height - 1` (we've reconnected + // to the known chain), + // - `seq` is empty (this is the first head we've ever seen — there + // are no ancestors to walk back to), + // - or we hit genesis (defensive — shouldn't happen on real chains). + let mut walked: Vec = Vec::new(); + let mut cursor = head_link; + loop { + // Parse both hashes eagerly so we don't fail later during commit + // (which would leave the sequence partially mutated). + let cursor_parent_typed = parse_hash::(&cursor.parent)?; + let _ = parse_hash::(&cursor.hash)?; + let parent_height = cursor.height.saturating_sub(1); + let reconnected = cursor.height == 0 + || seq.is_empty() + || seq + .get_block(parent_height, &cursor_parent_typed) + .is_some(); + let cursor_height = cursor.height; + walked.push(cursor); + if reconnected { + if cursor_height == 0 { + tracing::warn!("Re-org walk reached genesis without reconnecting"); + } + break; + } + let parent_ref = BlockReference::Hash(cursor_parent_typed); + cursor = data_provider.fetch_block_link(&parent_ref).await?; + } + + // Walk fully succeeded — now commit. Trim BEFORE inserts so a deep + // walk-back never evicts entries we're about to add this round. We use + // `pop_first` for one tree descent per eviction instead of + // `keys().next() + remove`. + while live_tokens.len() > history { + if live_tokens.pop_first().is_none() { + break; + } + } + + // Commit chain segments to `seq` in chain order (oldest-first), so the + // linkage check inside `seq.append` always sees parents before children. + for link in walked.iter().rev() { + let parent_h = parse_hash::(&link.parent)?; + let hash_h = parse_hash::(&link.hash)?; + seq.append(link.height, parent_h, hash_h); + } + + // Emit oldest-first so the archiver re-publishes in chain order. For each + // height, decide whether this is a fresh emission (no prior token) or a + // re-org replacement (prior token at this height with a different hash — + // cancel it before issuing the new job). + for link in walked.into_iter().rev() { + let token = match live_tokens.entry(link.height) { + Entry::Occupied(mut e) => { + let (existing_hash, existing_token) = e.get(); + if existing_hash == &link.hash { + // Unreachable in practice — the idempotency check at the + // top returns Ok for identical (height, hash). If we ever + // get here, reuse the existing token rather than minting + // a new one that would orphan the in-flight archive. + existing_token.clone() + } else { + tracing::info!( + height = link.height, + previous_hash = %existing_hash, + new_hash = %link.hash, + "Re-org: cancelling prior block at this height" + ); + existing_token.cancel(); + let new_token = CancellationToken::new(); + *e.get_mut() = (link.hash.clone(), new_token.clone()); + new_token + } + } + Entry::Vacant(e) => { + let new_token = CancellationToken::new(); + e.insert((link.hash.clone(), new_token.clone())); + new_token + } + }; + let job = BlockJob { + height: Height { + height: link.height, + hash: Some(link.hash), + }, + cancel: token, + }; + if tx.send(job).await.is_err() { + return Err(anyhow::anyhow!("Receiver dropped; follower exiting")); + } + } + + Ok(()) +} + +fn parse_hash(s: &str) -> anyhow::Result { + B::BlockHash::from_str(s) + .map_err(|_| anyhow::anyhow!("Failed to parse block hash: {}", s)) +} + /// /// Provides next finalized blocks for Ethereum-like blockchains. /// Finalized Block is a block that agreed by the majority of the network and extremely unlikely to be replaced (reorganized) @@ -41,7 +315,7 @@ impl NextFinalizedBlock { #[async_trait] impl NextBlock for NextFinalizedBlock { - async fn next_blocks(&self) -> Result, BlockchainError> { + async fn next_blocks(&self) -> Result, BlockchainError> { let mut head = self.blockchain.subscribe_blocks().await?; let (tx, rx) = tokio::sync::mpsc::channel(2); let data_provider = self.data_provider.clone(); @@ -87,7 +361,9 @@ impl NextBlock for NextFinalizedBlock { // i.e., since we were adding missing (=older) blocks at the end, we need to go from the back to the front for h in next.iter().rev() { tracing::debug!("Finalized Height: {}", h.height); - if let Err(e) = tx.send(h.clone()).await { + // Finalized blocks are settled by definition — no re-org + // signal applies, so every job carries an untracked token. + if let Err(e) = tx.send(BlockJob::untracked(h.clone())).await { tracing::error!("Failed to send finalized height: {}", e); break; } @@ -98,3 +374,497 @@ impl NextBlock for NextFinalizedBlock { Ok(rx) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::blockchain::mock::{MockBlock, MockData, MockType}; + + /// Build a chain of `count` blocks starting at `from_height` with the + /// given `hash_prefix`. The first block's parent is `parent_of_first`. + fn populate_chain( + data: &MockData, + from_height: u64, + count: u64, + hash_prefix: &str, + parent_of_first: &str, + ) { + let mut parent = parent_of_first.to_string(); + for i in 0..count { + let height = from_height + i; + let hash = format!("{}{}", hash_prefix, height); + data.add_block(MockBlock { + height, + hash: hash.clone(), + parent: parent.clone(), + transactions: vec![], + }); + parent = hash; + } + } + + async fn drain(rx: &mut tokio::sync::mpsc::Receiver) -> Vec { + let mut out = Vec::new(); + while let Ok(item) = rx.try_recv() { + out.push(item); + } + out + } + + /// First head event seen: nothing in the sequence yet, so it should be + /// emitted as-is without any walk-back. + #[tokio::test] + async fn first_head_event_is_emitted_as_is() { + let data = Arc::new(MockData::new("MOCK")); + populate_chain(&data, 100, 3, "0xA", "0xroot"); + let mut seq = BlockSequence::::new(64); + let mut live_tokens = BTreeMap::new(); + let (tx, mut rx) = tokio::sync::mpsc::channel(8); + handle_head_event::( + &mut seq, + &mut live_tokens, + 64, + &data, + Height { height: 100, hash: Some("0xA100".to_string()) }, + &tx, + ) + .await + .unwrap(); + drop(tx); + let emitted = drain(&mut rx).await; + assert_eq!(emitted.len(), 1); + assert_eq!(emitted[0].height.height, 100); + assert_eq!(emitted[0].height.hash.as_deref(), Some("0xA100")); + assert!(!emitted[0].cancel.is_cancelled()); + } + + /// Sequential head events: each new head's parent matches the previous + /// head's hash. Each one is emitted on its own with no walk-back. + #[tokio::test] + async fn sequential_heads_emit_one_at_a_time() { + let data = Arc::new(MockData::new("MOCK")); + populate_chain(&data, 100, 3, "0xA", "0xroot"); + let mut seq = BlockSequence::::new(64); + let mut live_tokens = BTreeMap::new(); + let (tx, mut rx) = tokio::sync::mpsc::channel(8); + for h in 100..103 { + handle_head_event::( + &mut seq, + &mut live_tokens, + 64, + &data, + Height { height: h, hash: Some(format!("0xA{}", h)) }, + &tx, + ) + .await + .unwrap(); + } + drop(tx); + let emitted = drain(&mut rx).await; + let names: Vec<_> = emitted + .iter() + .map(|j| (j.height.height, j.height.hash.clone().unwrap())) + .collect(); + assert_eq!( + names, + vec![ + (100, "0xA100".to_string()), + (101, "0xA101".to_string()), + (102, "0xA102".to_string()), + ] + ); + // None of the unique heights should have cancelled tokens. + assert!(emitted.iter().all(|j| !j.cancel.is_cancelled())); + } + + /// Same-height re-org: after seeing block A at height 102, a new block B + /// arrives at the same height with the same parent. The original A's + /// token must be cancelled before B is emitted with its own fresh token. + #[tokio::test] + async fn same_height_reorg_cancels_prior_token_and_emits_replacement() { + let data = Arc::new(MockData::new("MOCK")); + // Original chain: 100 (A100) → 101 (A101) → 102 (A102). + populate_chain(&data, 100, 3, "0xA", "0xroot"); + // Re-org variant of 102 with a different hash but same parent (A101). + data.add_block(MockBlock { + height: 102, + hash: "0xB102".to_string(), + parent: "0xA101".to_string(), + transactions: vec![], + }); + + let mut seq = BlockSequence::::new(64); + let mut live_tokens = BTreeMap::new(); + let (tx, mut rx) = tokio::sync::mpsc::channel(8); + for h in 100..103 { + handle_head_event::( + &mut seq, + &mut live_tokens, + 64, + &data, + Height { height: h, hash: Some(format!("0xA{}", h)) }, + &tx, + ) + .await + .unwrap(); + } + // Capture the token issued for A102 — this is the one we expect the + // re-org to fire. + let a102_token = live_tokens.get(&102).unwrap().1.clone(); + assert!(!a102_token.is_cancelled()); + + // Now the re-org head arrives. + handle_head_event::( + &mut seq, + &mut live_tokens, + 64, + &data, + Height { height: 102, hash: Some("0xB102".to_string()) }, + &tx, + ) + .await + .unwrap(); + drop(tx); + + // The original A102 token must now be cancelled. + assert!(a102_token.is_cancelled(), "prior block token at height 102 must be cancelled on re-org"); + + // Last emission should be B102 with a fresh, uncancelled token. + let emitted = drain(&mut rx).await; + let last = emitted.last().unwrap(); + assert_eq!(last.height.height, 102); + assert_eq!(last.height.hash.as_deref(), Some("0xB102")); + assert!(!last.cancel.is_cancelled()); + } + + /// Deep re-org: heights 102..105 of the A-chain are replaced by a new + /// B-chain. The follower must cancel the prior token at each replaced + /// height (here only 102 and 103 had prior tokens — 104/105 are new). + #[tokio::test] + async fn deep_reorg_walks_back_and_emits_new_tail_in_order() { + let data = Arc::new(MockData::new("MOCK")); + // Original A-chain: 100..=103, all with prefix "0xA". + populate_chain(&data, 100, 4, "0xA", "0xroot"); + // B-chain forks off A101: 102..=105 with prefix "0xB", parent of B102 = A101. + populate_chain(&data, 102, 4, "0xB", "0xA101"); + + let mut seq = BlockSequence::::new(64); + let mut live_tokens = BTreeMap::new(); + let (tx, mut rx) = tokio::sync::mpsc::channel(32); + // Replay the A-chain into the sequence. + for h in 100..=103 { + handle_head_event::( + &mut seq, + &mut live_tokens, + 64, + &data, + Height { height: h, hash: Some(format!("0xA{}", h)) }, + &tx, + ) + .await + .unwrap(); + } + let a102_token = live_tokens.get(&102).unwrap().1.clone(); + let a103_token = live_tokens.get(&103).unwrap().1.clone(); + let _ = drain(&mut rx).await; // discard original emissions + + // Now the new head jumps straight to B105. + handle_head_event::( + &mut seq, + &mut live_tokens, + 64, + &data, + Height { height: 105, hash: Some("0xB105".to_string()) }, + &tx, + ) + .await + .unwrap(); + drop(tx); + + // Both pre-existing tokens at the re-orged heights must be cancelled. + assert!(a102_token.is_cancelled(), "A102 token must be cancelled on deep re-org"); + assert!(a103_token.is_cancelled(), "A103 token must be cancelled on deep re-org"); + + let emitted = drain(&mut rx).await; + let names: Vec<_> = emitted + .iter() + .map(|j| (j.height.height, j.height.hash.clone().unwrap())) + .collect(); + // Re-org tail: B102, B103, B104, B105 in chain order. + assert_eq!( + names, + vec![ + (102, "0xB102".to_string()), + (103, "0xB103".to_string()), + (104, "0xB104".to_string()), + (105, "0xB105".to_string()), + ] + ); + // The replacements should all have fresh, uncancelled tokens. + assert!(emitted.iter().all(|j| !j.cancel.is_cancelled())); + } + + /// A→B→A revert: a chain that re-orgs from A to B and then back to A + /// must re-emit A so consumers learn the chain reverted. Previously the + /// `seq.get_block` idempotency check matched any historical hash at the + /// height, silently swallowing the reversion. + #[tokio::test] + async fn revert_back_to_previous_block_is_re_emitted() { + let data = Arc::new(MockData::new("MOCK")); + // Both A102 and B102 share parent A101 — A and B are siblings at + // height 102. + populate_chain(&data, 100, 3, "0xA", "0xroot"); + data.add_block(MockBlock { + height: 102, + hash: "0xB102".to_string(), + parent: "0xA101".to_string(), + transactions: vec![], + }); + + let mut seq = BlockSequence::::new(64); + let mut live_tokens = BTreeMap::new(); + let (tx, mut rx) = tokio::sync::mpsc::channel(8); + + // Emit A100..A102, then re-org to B102, then revert back to A102. + for h in 100..103 { + handle_head_event::( + &mut seq, + &mut live_tokens, + 64, + &data, + Height { height: h, hash: Some(format!("0xA{}", h)) }, + &tx, + ) + .await + .unwrap(); + } + handle_head_event::( + &mut seq, + &mut live_tokens, + 64, + &data, + Height { height: 102, hash: Some("0xB102".to_string()) }, + &tx, + ) + .await + .unwrap(); + // Capture B102's token — when A returns it should be cancelled. + let b102_token = live_tokens.get(&102).unwrap().1.clone(); + assert!(!b102_token.is_cancelled()); + + // Chain reverts to A102. + handle_head_event::( + &mut seq, + &mut live_tokens, + 64, + &data, + Height { height: 102, hash: Some("0xA102".to_string()) }, + &tx, + ) + .await + .unwrap(); + drop(tx); + + assert!( + b102_token.is_cancelled(), + "B102 token must be cancelled when the chain reverts back to A" + ); + + let emitted = drain(&mut rx).await; + let last = emitted.last().unwrap(); + assert_eq!(last.height.height, 102); + assert_eq!( + last.height.hash.as_deref(), + Some("0xA102"), + "A102 must be re-emitted on reversion" + ); + assert!(!last.cancel.is_cancelled()); + // The latest live hash at height 102 must reflect A again. + assert_eq!(live_tokens.get(&102).unwrap().0, "0xA102"); + } + + /// Walk-back atomicity: if fetching an ancestor fails partway through a + /// deep re-org walk, neither `seq` nor `live_tokens` should be mutated. + /// The next head event must be able to re-validate from scratch. + #[tokio::test] + async fn walk_back_fetch_failure_leaves_seq_untouched() { + let data = Arc::new(MockData::new("MOCK")); + // Seed the original chain so the follower has something to re-org + // against. + populate_chain(&data, 100, 2, "0xA", "0xroot"); + + let mut seq = BlockSequence::::new(64); + let mut live_tokens = BTreeMap::new(); + let (tx, mut rx) = tokio::sync::mpsc::channel(8); + + for h in 100..102 { + handle_head_event::( + &mut seq, + &mut live_tokens, + 64, + &data, + Height { height: h, hash: Some(format!("0xA{}", h)) }, + &tx, + ) + .await + .unwrap(); + } + // Snapshot bookkeeping BEFORE the doomed walk. CancellationToken has + // no PartialEq, so we compare the (height, hash) pairs that drive + // the bookkeeping correctness invariant. + let snapshot = |t: &BTreeMap| -> Vec<(u64, String)> { + t.iter().map(|(h, (hash, _))| (*h, hash.clone())).collect() + }; + let live_tokens_before = snapshot(&live_tokens); + // Discard the original emissions. + let _ = drain(&mut rx).await; + + // Add only the TOP of a B-chain to MockData: the walk-back from B105 + // will succeed for B105 but fail at B104 (never added). The middle + // ancestors don't exist as far as the data provider is concerned. + data.add_block(MockBlock { + height: 105, + hash: "0xB105".to_string(), + parent: "0xB104".to_string(), + transactions: vec![], + }); + let result = handle_head_event::( + &mut seq, + &mut live_tokens, + 64, + &data, + Height { height: 105, hash: Some("0xB105".to_string()) }, + &tx, + ) + .await; + assert!(result.is_err(), "walk-back must fail when an ancestor is unfetchable"); + + // `seq` and `live_tokens` must reflect ONLY the pre-walk A-chain + // state — none of the doomed B-chain heights leaked in. + assert!( + seq.get_block(105, &"0xB105".to_string()).is_none(), + "B105 must not be committed to seq after a failed walk-back" + ); + assert_eq!( + snapshot(&live_tokens), + live_tokens_before, + "live_tokens must be unchanged after a failed walk-back" + ); + drop(tx); + let emitted = drain(&mut rx).await; + assert!( + emitted.is_empty(), + "no BlockJob should be emitted from a failed walk-back" + ); + } + + /// Deep walk-back trim: when the segment to emit exceeds `history`, the + /// trim must NOT evict tokens for jobs we're about to install in the same + /// call. Previously the trim ran AFTER the emit loop and could drop the + /// oldest entries even though they were just added. + #[tokio::test] + async fn deep_walk_back_does_not_evict_just_inserted_tokens() { + let data = Arc::new(MockData::new("MOCK")); + // 30 sequential A-blocks; we'll pre-warm with 100..104 then trigger + // a walk-back from height 129 that walks 105..129 (25 heights). + populate_chain(&data, 100, 30, "0xA", "0xroot"); + + let mut seq = BlockSequence::::new(64); + let mut live_tokens = BTreeMap::new(); + let (tx, mut rx) = tokio::sync::mpsc::channel(64); + // Pre-warm: emit 100..104 as separate events so seq isn't empty when + // the deep event arrives (otherwise the walk-back short-circuits on + // the first iteration because there's nothing to reconnect to). + for h in 100..=104 { + handle_head_event::( + &mut seq, + &mut live_tokens, + 8, + &data, + Height { height: h, hash: Some(format!("0xA{}", h)) }, + &tx, + ) + .await + .unwrap(); + } + let _ = drain(&mut rx).await; // discard the warmup emissions + + // Jump to height 129. Walk-back must traverse 128..105 by parent + // hash, reconnecting at A104 (the youngest entry in seq). That's a + // 25-height segment with `history=8` — a post-emit trim would have + // silently dropped the oldest 22 entries (heights 105..126). The + // new pre-emit trim must NOT. + handle_head_event::( + &mut seq, + &mut live_tokens, + 8, + &data, + Height { height: 129, hash: Some("0xA129".to_string()) }, + &tx, + ) + .await + .unwrap(); + drop(tx); + + let emitted = drain(&mut rx).await; + assert_eq!(emitted.len(), 25, "all 25 walked-back heights must emit"); + for job in &emitted { + let h = job.height.height; + let entry = live_tokens + .get(&h) + .unwrap_or_else(|| panic!("missing live_tokens entry for emitted height {}", h)); + assert_eq!( + entry.0, + job.height.hash.clone().unwrap(), + "live_tokens hash must match emitted hash at height {}", + h + ); + assert!( + !job.cancel.is_cancelled(), + "freshly-emitted job at height {} must not carry a cancelled token", + h + ); + } + } + + /// Idempotency: a head event for a block already in the sequence (e.g., + /// the broker re-sending the current tip on reconnect) is silently + /// dropped — and the existing token is not cancelled. + #[tokio::test] + async fn duplicate_head_event_is_silently_dropped() { + let data = Arc::new(MockData::new("MOCK")); + populate_chain(&data, 100, 2, "0xA", "0xroot"); + + let mut seq = BlockSequence::::new(64); + let mut live_tokens = BTreeMap::new(); + let (tx, mut rx) = tokio::sync::mpsc::channel(8); + handle_head_event::( + &mut seq, + &mut live_tokens, + 64, + &data, + Height { height: 100, hash: Some("0xA100".to_string()) }, + &tx, + ) + .await + .unwrap(); + let a100_token = live_tokens.get(&100).unwrap().1.clone(); + // Re-emit the same head. + handle_head_event::( + &mut seq, + &mut live_tokens, + 64, + &data, + Height { height: 100, hash: Some("0xA100".to_string()) }, + &tx, + ) + .await + .unwrap(); + drop(tx); + + let emitted = drain(&mut rx).await; + assert_eq!(emitted.len(), 1, "duplicate head should not re-emit"); + // No cancellation should happen on a duplicate. + assert!(!a100_token.is_cancelled()); + } +} diff --git a/src/command/archive.rs b/src/command/archive.rs index 1219b38..c4c7e21 100644 --- a/src/command/archive.rs +++ b/src/command/archive.rs @@ -2,6 +2,7 @@ use std::marker::PhantomData; use std::str::FromStr; use anyhow::anyhow; use async_trait::async_trait; +use tokio_util::sync::CancellationToken; use crate::{ archiver::{ArchiveAll, Archiver}, args::Args, @@ -35,11 +36,15 @@ impl CommandExecutor for ArchiveCommand CommandExecutor for FixCommand { ..self.tx_options.clone() }; let missing = self.archiver.target.find_incomplete_tables(range, &options).await?; + // `fix` runs against settled archive state — no re-org signal applies. + let cancel = CancellationToken::new(); for (range, kinds) in missing { if shutdown.is_signalled() { break; @@ -61,7 +64,7 @@ impl CommandExecutor for FixCommand { } tracing::info!(range = %chunk, "Fixing chunk"); if !dry_run { - self.archiver.archive(chunk, RunMode::Fix, None, &options).await?; + self.archiver.archive(chunk, RunMode::Fix, None, &options, &cancel).await?; } } } diff --git a/src/command/stream.rs b/src/command/stream.rs index b1ec102..4a28f75 100644 --- a/src/command/stream.rs +++ b/src/command/stream.rs @@ -2,7 +2,7 @@ use std::marker::PhantomData; use std::sync::Arc; use async_trait::async_trait; use crate::{ - archiver::{ArchiveAll, Archiver}, + archiver::{ArchiveAll, Archiver, ScanResume, StreamResume}, args::Args, blockchain::{ connection::Blockchain, @@ -11,83 +11,123 @@ use crate::{ command::CommandExecutor, global, notify::RunMode, - storage::ScanTarget + storage::{ScanTarget, WriteTarget} }; -use anyhow::Result; +use anyhow::{anyhow, Result}; use crate::archiver::datakind::DataOptions; -use crate::archiver::range::{Height, Range}; use crate::args::Follow; use crate::blockchain::BlockchainData; +use crate::blockchain::next_block::ReorgAwareFollower; use crate::notify::Maturity; /// /// Provides `stream` command. -/// It appends fresh blocks one by one to the archive +/// It appends fresh blocks one by one to the archive. /// -#[derive(Clone)] -pub struct StreamCommand { +/// Generic over any [`WriteTarget`], so it serves both file-based targets +/// (Avro, JSON) and streaming targets (Pulsar). The optional [`StreamResume`] +/// strategy lets file-based callers opt into `--continue` behaviour while +/// streaming targets simply pass `None` — they don't have a tail-scan +/// capability in the v1 implementation. +pub struct StreamCommand { b: PhantomData, blockchain: Arc, - continue_blocks: Option, archiver: Archiver, data_options: DataOptions, follow: Follow, + /// `Some` when the user passed `--continue` *and* the target supports + /// resume. Streaming targets always carry `None`; for those, `--continue` + /// is rejected at construction time. + resume: Option>, } -impl StreamCommand { - pub async fn new(config: &Args, - archiver: Archiver - ) -> Result { - let blockchain = Arc::new(Blockchain::new(&config.connection, config.as_dshackle_blockchain()?, config.get_blockchain()?.code()).await?); +/// Number of blocks the `--continue` tail scan rewinds before live streaming +/// resumes. Matches the original (pre-refactor) hardcoded value so the +/// observable behaviour of `stream --continue` is unchanged. +const CONTINUE_TAIL_BLOCKS: u64 = 100; - let continue_blocks = if config.continue_last { - Some(100) - } else { - None - }; +/// Build the [`DataOptions`] used for the stream command — same as +/// `DataOptions::from(args)` but with `overwrite: false` so simultaneous +/// streams (e.g., one Head + one Finalized) don't clobber each other's files. +fn stream_data_options(config: &Args) -> DataOptions { + DataOptions { + overwrite: false, + ..DataOptions::from(config) + } +} - let data_options = DataOptions { - // keep existing files. ex when two stream are run simultaneously (one for Head, and one for Finalized) - overwrite: false, - ..DataOptions::from(config) - }; - let follow = config.follow.clone(); +impl StreamCommand { + /// Build a stream command for any [`WriteTarget`], with no resume support. + /// + /// Used by streaming targets (Pulsar). Rejects `--continue` at startup + /// because the target can't enumerate existing data. The Pulsar dispatch + /// path in `main` already rejects `--continue` upfront with a clearer + /// message; this check is the fallback that catches any future write-only + /// target wired in the same way. + pub async fn new(config: &Args, archiver: Archiver) -> Result { + if config.continue_last { + return Err(anyhow!( + "--continue is not supported by the selected target (no tail-scan capability)" + )); + } + Self::build(config, archiver, stream_data_options(config), None).await + } + async fn build( + config: &Args, + archiver: Archiver, + data_options: DataOptions, + resume: Option>, + ) -> Result { + let blockchain = Arc::new( + Blockchain::new( + &config.connection, + config.as_dshackle_blockchain()?, + config.get_blockchain()?.code(), + ) + .await?, + ); + let follow = config.follow.clone(); Ok(Self { b: PhantomData, blockchain, - continue_blocks, archiver, data_options, follow, + resume, }) } +} - /// - /// Ensures that the last N blocks are archived, where N is `self.continue_blocks`. - async fn ensure_continued(&self, height: Height) -> Result<()> { - if let Some(len) = self.continue_blocks { - let range = Range::up_to(len, &Range::Single(height)); - let options = self.data_options.clone(); - let missing = self.archiver.target.find_incomplete_tables(range, &options).await?; - for (range, kinds) in missing { - let range_opts= options.clone().only_include(&kinds); - for height in range.iter().collect::>() { - self.archiver.archive( - Height::from(height), - RunMode::Stream, - None, - &range_opts - ).await?; - } - } - } - Ok(()) +impl StreamCommand +where + B: BlockchainTypes + 'static, + TS: ScanTarget + 'static, +{ + /// Build a stream command that honours `--continue` against a target that + /// can list existing data ([`ScanTarget`]). When `--continue` is not set + /// the resume strategy is left empty and the behaviour matches + /// [`StreamCommand::new`]. + pub async fn new_with_resume(config: &Args, archiver: Archiver) -> Result { + let data_options = stream_data_options(config); + // The resumer shares the live stream's DataOptions so re-archived + // tail blocks land with the same `overwrite: false` semantics as + // fresh ones. + let resume: Option> = if config.continue_last { + Some(ScanResume::boxed( + archiver.clone(), + CONTINUE_TAIL_BLOCKS, + data_options.clone(), + )) + } else { + None + }; + Self::build(config, archiver, data_options, resume).await } } #[async_trait] -impl CommandExecutor for StreamCommand { +impl CommandExecutor for StreamCommand { async fn execute(&self) -> Result<()> { @@ -96,9 +136,15 @@ impl CommandExecutor for StreamCommand Maturity::Finalized, }; - let heights = match self.follow { + let heights: Box = match self.follow { Follow::Latest => { - Box::new(self.blockchain.clone()) + // Wrap the raw head subscription in the re-org aware follower + // so live re-orgs (same-height and deep) get re-emitted with + // proper chain order. See `ReorgAwareFollower` for details. + Box::new(ReorgAwareFollower::::new( + self.blockchain.clone(), + self.archiver.data_provider.clone(), + )) } Follow::Finalized => { self.archiver.data_provider.next_finalized_blocks()? @@ -108,7 +154,7 @@ impl CommandExecutor for StreamCommand CommandExecutor for StreamCommand { crate::progress::resume(); - if let Some(height) = next { + if let Some(job) = next { // when we have learned the latest height, we ensure that the last N blocks are archived; but just once if !continued { - let up_to_height = height.clone(); - // we ignore the error here because the new blocks should be more important - // and if it failed here then the Fix command can fix it later - let _ = self.ensure_continued(up_to_height).await; + if let Some(resume) = &self.resume { + let up_to_height = job.height.clone(); + // we ignore the error here because the new blocks should be more important + // and if it failed here then the Fix command can fix it later + let _ = resume.ensure_continued(up_to_height).await; + } continued = true; } - tracing::info!("Archive block: {} {:?}", height.height, height.hash); - self.archiver.archive(height, RunMode::Stream, Some(maturity.clone()), &self.data_options).await?; + tracing::info!("Archive block: {} {:?}", job.height.height, job.height.hash); + // The follower owns the cancel token for this job; if a re-org replaces + // the block mid-archive it will fire the token and the archiver path + // unwinds cooperatively (see `ProcessOutcome::Cancelled`). + self.archiver.archive(job.height, RunMode::Stream, Some(maturity.clone()), &self.data_options, &job.cancel).await?; } else { stop = true; } diff --git a/src/formats/avro/mod.rs b/src/formats/avro/mod.rs index 62c8b62..a401fb4 100644 --- a/src/formats/avro/mod.rs +++ b/src/formats/avro/mod.rs @@ -217,6 +217,7 @@ mod tests { parent_id: Some("0xparent".to_string()), tx_index: None, tx_id: None, + tx_count: None, fields: vec![Field::BlockJson(b"{}".to_vec())], } } @@ -267,6 +268,7 @@ mod tests { parent_id: None, tx_index: Some(3), tx_id: Some("0xtx".to_string()), + tx_count: None, fields: vec![ Field::TxJson(b"{}".to_vec()), Field::TxRaw(vec![1, 2, 3]), diff --git a/src/formats/json.rs b/src/formats/json.rs index a36a699..6bdebea 100644 --- a/src/formats/json.rs +++ b/src/formats/json.rs @@ -202,6 +202,7 @@ mod tests { parent_id: Some("0xparent".to_string()), tx_index: tx_id.map(|_| 0), tx_id: tx_id.map(|s| s.to_string()), + tx_count: None, fields, } } diff --git a/src/formats/mod.rs b/src/formats/mod.rs index ff29d6c..53ea33b 100644 --- a/src/formats/mod.rs +++ b/src/formats/mod.rs @@ -11,3 +11,4 @@ pub mod avro; pub mod json; +pub mod stream; diff --git a/src/formats/stream.rs b/src/formats/stream.rs new file mode 100644 index 0000000..ae727de --- /dev/null +++ b/src/formats/stream.rs @@ -0,0 +1,760 @@ +// Copyright 2026 EmeraldPay Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. + +//! Streaming-target row encoding. +//! +//! Where [`crate::formats::json`] turns an [`ArchiveRow`] into per-field *files*, +//! this module turns the same row into per-field *messages* destined for one of +//! the streaming brokers (Pulsar today, Kafka next). +//! +//! Each [`Field`] variant maps to a stable topic label that the storage backend +//! appends to the user-supplied topic prefix, producing the per-field topic +//! name. The message payload is a JSON object — an [`Entry`] — that wraps the +//! original node response (or hex string for raw transactions) together with +//! the metadata a consumer needs to route, filter or dedup the message without +//! relying on broker headers. Header-only metadata used to be enough, but +//! several downstream sinks (notably Pulsar IO connectors) only forward the +//! message body, so the wrapping struct keeps the contract self-describing +//! regardless of what the consumer sees. +//! +//! A small subset of metadata is *also* attached as broker properties +//! (`dedup-key`, `timestamp`, `height`, `block-id`) so brokers and lightweight +//! consumers (server-side selectors, simple log tailers) can route and dedup +//! without parsing JSON. + +use std::collections::HashMap; + +use chrono::{DateTime, Utc}; +use serde::{Serialize, Serializer}; +use serde_json::value::{to_raw_value, RawValue}; + +use crate::archiver::datakind::{DataKind, DataOptions}; +use crate::record::{ArchiveRow, BlockchainType, Field}; + +/// The set of topic labels actually published for the given +/// `(blockchain, data_options)` combination. +/// +/// Two filters apply: +/// +/// 1. **Blockchain shape.** Bitcoin has no receipts, no uncles, and no +/// traces, so those topics are never created for a Bitcoin run. Ethereum +/// can publish to every label. +/// 2. **User selection.** `--tables` controls whether block / tx / trace +/// topics are created at all; `--fields.trace` further narrows the trace +/// topic set to `calls` / `statediff`. +/// +/// Returned labels follow a stable order (blocks → uncles → tx → traces), +/// so callers can rely on consistent iteration order for logging. +pub fn topic_labels_for( + blockchain: BlockchainType, + options: &DataOptions, +) -> Vec<&'static str> { + let mut labels: Vec<&'static str> = Vec::new(); + + // Block-kind topics. + if options.include_block() { + labels.push("blocks"); + if matches!(blockchain, BlockchainType::Ethereum) { + labels.push("blocks-uncles"); + } + } + + // Transaction-kind topics. Bitcoin produces tx-json + tx-raw; Ethereum + // additionally produces tx-receipts. + if options.include_tx() { + labels.push("tx-json"); + labels.push("tx-raw"); + if matches!(blockchain, BlockchainType::Ethereum) { + labels.push("tx-receipts"); + } + } + + // Trace-kind topics. Bitcoin has no traces — even if the user passes + // `--tables traces`, we suppress the topic creation here so a misconfig + // doesn't create dead Pulsar topics. For Ethereum, each sub-field + // (`calls` / `stateDiff`) is created only when its corresponding flag is + // set on `TraceOptions`. + if options.include_trace() && matches!(blockchain, BlockchainType::Ethereum) { + if let Some(trace) = options.trace.as_ref() { + if trace.include_trace { + labels.push("trace-calls"); + } + if trace.include_state_diff { + labels.push("trace-statediff"); + } + } + } + + labels +} + +/// One message destined for a single per-field topic. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StreamMessage { + /// Topic label (one of the labels [`topic_labels_for`] would produce + /// for the running blockchain). The backend builds the full topic name + /// as `-`. + pub field: &'static str, + /// JSON bytes of the serialized [`Entry`] — the node response wrapped + /// alongside its routing metadata. + pub payload: Vec, + /// Partition key. Always the stringified block height so every message for + /// a given block — including same-height re-orgs — lands in the same + /// partition and is consumed in publish order. + pub partition_key: String, + /// Broker-side properties retained for fast filtering and dedup at the + /// broker layer. Full metadata also lives inside the payload [`Entry`], + /// so consumers that only see the body still have everything they need. + pub properties: HashMap, +} + +/// Envelope written to the wire as the message payload. +/// +/// Wraps the original node response (or, for raw transactions, the +/// chain-formatted hex string) together with the metadata a consumer needs +/// to route, filter or dedup without parsing the inner value. Compound field +/// names follow the same camelCase convention as +/// [`crate::notify::Notification`], so a downstream that already speaks one +/// Dshackle Archive JSON flavour stays consistent across both. +/// +/// `value` is held as a [`RawValue`] so the original node JSON is embedded +/// byte-for-byte instead of being parsed-and-reserialized. +#[derive(Debug, Serialize)] +struct Entry<'a> { + /// Blockchain id (`ETH`, `BTC`, …) — mirrors `blockchain_id` on the row. + blockchain: &'a str, + /// Block timestamp as reported by the node, serialized as RFC 3339. + timestamp: DateTime, + /// Logical table this row belongs to: `blocks`, `transactions`, or + /// `traces`. Plural matches the canonical table naming used for the + /// Avro files and the JSON layout's per-kind directories. Serialized + /// via [`serialize_table`] to call [`DataKind::table`] directly, + /// decoupling Entry's on-the-wire format from any future change to + /// `DataKind`'s default `serde(rename)` (which is also used by + /// `Notification` and could drift). + #[serde(serialize_with = "serialize_table")] + table: DataKind, + /// Field label — same value as the enclosing message's topic suffix. + field: &'static str, + /// Block height. + height: u64, + /// Block hash. Distinguishes same-height re-orgs. + #[serde(rename = "blockId")] + block_id: &'a str, + /// Parent block hash. + #[serde(rename = "parentId", skip_serializing_if = "Option::is_none")] + parent_id: Option<&'a str>, + /// Transaction index within the block. Present on tx/trace rows. + #[serde(rename = "txIndex", skip_serializing_if = "Option::is_none")] + tx_index: Option, + /// Total number of transactions in the block — pairs with `txIndex` + /// to give a tx/trace consumer its position (`N` of `txCount`), and + /// surfaces the block's tx volume on block rows. + #[serde(rename = "txCount", skip_serializing_if = "Option::is_none")] + tx_count: Option, + /// Transaction id (hash). Present on tx/trace rows. + #[serde(rename = "txId", skip_serializing_if = "Option::is_none")] + tx_id: Option<&'a str>, + /// Uncle index. Present on Ethereum uncle messages only. + #[serde(rename = "uncleIndex", skip_serializing_if = "Option::is_none")] + uncle_index: Option, + /// Original node response. JSON for `*.json` fields; a JSON string + /// (chain-prefixed hex) for raw transactions. + value: &'a RawValue, +} + +/// Convert an [`ArchiveRow`] into the per-field messages it produces under the +/// streaming layout. The caller (the broker writer) decides which topic each +/// `field` maps to. +pub fn encode_row(row: &ArchiveRow) -> Vec { + row.fields + .iter() + .filter_map(|f| encode_field(row, f)) + .collect() +} + +fn encode_field(row: &ArchiveRow, field: &Field) -> Option { + // Per-variant data needed to build the message: the JSON-encoded value to + // embed, whether this field is keyed by `tx_id`, and the uncle index when + // applicable. The topic label itself comes from [`Field::name`] — there's + // no per-variant string here, so a new Field variant gets a label "for + // free" once it's added to [`Field::name`]. + let (value, tx_keyed, uncle_index): (Box, bool, Option) = match field { + Field::BlockJson(bytes) => (raw_value_from_bytes(bytes)?, false, None), + Field::Uncle { index, json } => (raw_value_from_bytes(json)?, false, Some(*index)), + Field::TxJson(bytes) => (raw_value_from_bytes(bytes)?, true, None), + Field::TxRaw(bytes) => (raw_value_from_tx_raw(bytes, row.blockchain_type), true, None), + Field::Receipt(bytes) => (raw_value_from_bytes(bytes)?, true, None), + // From/To duplicate values already inside the tx JSON; intentionally + // skipped, matching the JSON-file layout. They still carry a name on + // [`Field`] for completeness, just no producer is registered for them. + Field::From(_) | Field::To(_) => return None, + Field::Trace(bytes) => (raw_value_from_bytes(bytes)?, true, None), + Field::StateDiff(bytes) => (raw_value_from_bytes(bytes)?, true, None), + }; + + let label = field.topic_label(); + let tx_id = if tx_keyed { row.tx_id.as_deref() } else { None }; + + let entry = Entry { + blockchain: &row.blockchain_id, + timestamp: row.timestamp, + table: row.kind, + field: label, + height: row.height, + block_id: &row.block_id, + parent_id: row.parent_id.as_deref(), + tx_index: row.tx_index, + tx_count: row.tx_count, + tx_id, + uncle_index, + value: &value, + }; + + let payload = match serde_json::to_vec(&entry) { + Ok(b) => b, + Err(e) => { + tracing::warn!(field = label, error = %e, "Failed to serialize stream entry"); + return None; + } + }; + + let mut properties = HashMap::new(); + properties.insert("timestamp".to_string(), row.timestamp.to_rfc3339()); + properties.insert("height".to_string(), row.height.to_string()); + properties.insert("block-id".to_string(), row.block_id.clone()); + properties.insert( + "dedup-key".to_string(), + dedup_key(row, label, tx_id, uncle_index), + ); + + Some(StreamMessage { + field: label, + payload, + partition_key: row.height.to_string(), + properties, + }) +} + +/// Serialize a [`DataKind`] as its plural table name (see +/// [`DataKind::table`]). Used on [`Entry::table`] via +/// `#[serde(serialize_with = ...)]` so the wire format is pinned to +/// `DataKind::table()` rather than the derive's `#[serde(rename)]`. +fn serialize_table(kind: &DataKind, ser: S) -> Result { + ser.serialize_str(kind.table()) +} + +/// Wrap bytes from a node JSON response as a [`RawValue`] without +/// reserializing. Returns `None` if the bytes are not valid UTF-8 or not +/// valid JSON — that shouldn't happen for live node data, but a single +/// corrupt row shouldn't poison the whole stream. +fn raw_value_from_bytes(bytes: &[u8]) -> Option> { + let s = std::str::from_utf8(bytes).ok()?; + RawValue::from_string(s.to_string()).ok() +} + +/// Build a [`RawValue`] containing a JSON string of the hex-encoded raw tx, +/// with the chain-appropriate prefix (`0x` for Ethereum, none for Bitcoin). +fn raw_value_from_tx_raw(bytes: &[u8], blockchain_type: BlockchainType) -> Box { + let hex_str = hex::encode(bytes); + let s = match blockchain_type { + BlockchainType::Ethereum => format!("0x{}", hex_str), + BlockchainType::Bitcoin => hex_str, + }; + // Infallible: a `String` always serializes to a valid JSON value. + to_raw_value(&s).expect("string serializes to RawValue") +} + +/// Deterministic key used by consumers to dedup re-emitted messages (e.g., +/// after a future restart-with-resume run that has to re-publish the tail of +/// a partial block). +/// +/// Format: `:[:tx-][:uncle-]`. +/// +/// **Why `block-id` and not `height`:** a chain re-org produces a *different* +/// block at an existing height, frequently with overlapping tx hashes. From a +/// consumer's perspective those are not duplicates — they replace the previous +/// block's state. Keying on the block hash makes the dedup key +/// reorg-correct: same height with a different block id gets a different +/// dedup key and is preserved, while a true re-emission of the *same* +/// (block-id, tx-id) pair correctly collapses. +/// +/// **Why no `blockchain`:** each topic is single-blockchain by construction +/// (different chains run as separate processes and write to disjoint topic +/// prefixes), so a blockchain prefix here would be dead weight. +fn dedup_key( + row: &ArchiveRow, + label: &'static str, + tx_id: Option<&str>, + uncle_index: Option, +) -> String { + let mut parts: Vec = vec![label.to_string(), row.block_id.clone()]; + if let Some(tx_id) = tx_id { + parts.push(format!("tx-{}", tx_id)); + } + if let Some(i) = uncle_index { + parts.push(format!("uncle-{}", i)); + } + parts.join(":") +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::{TimeZone, Utc}; + use serde_json::Value; + use std::collections::HashSet; + + use crate::archiver::datakind::DataKind; + use crate::record::BlockchainType; + + fn row(kind: DataKind, tx_id: Option<&str>, fields: Vec) -> ArchiveRow { + ArchiveRow { + kind, + blockchain_type: BlockchainType::Ethereum, + blockchain_id: "ETH".to_string(), + archive_ts: Utc::now(), + height: 100, + block_id: "0xblock".to_string(), + timestamp: Utc.timestamp_millis_opt(0).unwrap(), + parent_id: Some("0xparent".to_string()), + tx_index: tx_id.map(|_| 7), + tx_id: tx_id.map(|s| s.to_string()), + tx_count: Some(12), + fields, + } + } + + fn parse(payload: &[u8]) -> Value { + serde_json::from_slice(payload).expect("payload is valid JSON") + } + + #[test] + fn block_row_emits_blocks_and_uncle_messages() { + let r = row( + DataKind::Blocks, + None, + vec![ + Field::BlockJson(b"{\"h\":1}".to_vec()), + Field::Uncle { index: 0, json: b"{\"u\":0}".to_vec() }, + Field::Uncle { index: 1, json: b"{\"u\":1}".to_vec() }, + ], + ); + let msgs = encode_row(&r); + assert_eq!(msgs.len(), 3); + assert_eq!(msgs[0].field, "blocks"); + assert_eq!(msgs[1].field, "blocks-uncles"); + let u0 = parse(&msgs[1].payload); + assert_eq!(u0["uncleIndex"], 0); + let u1 = parse(&msgs[2].payload); + assert_eq!(u1["uncleIndex"], 1); + } + + #[test] + fn tx_row_emits_per_field_messages_with_metadata() { + let r = row( + DataKind::Transactions, + Some("0xabc"), + vec![ + Field::TxJson(b"{\"a\":1}".to_vec()), + Field::TxRaw(vec![0xde, 0xad, 0xbe, 0xef]), + Field::Receipt(b"{\"r\":1}".to_vec()), + Field::From("0xfrom".to_string()), + Field::To("0xto".to_string()), + ], + ); + let msgs = encode_row(&r); + // From/To are intentionally skipped. + assert_eq!(msgs.len(), 3); + let tx_msg = msgs.iter().find(|m| m.field == "tx-json").unwrap(); + assert_eq!(tx_msg.partition_key, "100"); + let entry = parse(&tx_msg.payload); + assert_eq!(entry["blockchain"], "ETH"); + assert_eq!(entry["table"], "transactions"); + assert_eq!(entry["field"], "tx-json"); + assert_eq!(entry["height"], 100); + assert_eq!(entry["blockId"], "0xblock"); + assert_eq!(entry["parentId"], "0xparent"); + assert_eq!(entry["txIndex"], 7); + assert_eq!(entry["txId"], "0xabc"); + // Inner value is embedded as JSON, not as a string. + assert_eq!(entry["value"], serde_json::json!({"a": 1})); + // Header subset still surfaced for broker-side filtering. + assert_eq!(tx_msg.properties.get("height").unwrap(), "100"); + assert_eq!(tx_msg.properties.get("block-id").unwrap(), "0xblock"); + assert_eq!( + tx_msg.properties.get("dedup-key").unwrap(), + "tx-json:0xblock:tx-0xabc" + ); + // Properties no longer carry the full envelope. + assert!(tx_msg.properties.get("blockchain").is_none()); + assert!(tx_msg.properties.get("table").is_none()); + assert!(tx_msg.properties.get("field").is_none()); + assert!(tx_msg.properties.get("tx-index").is_none()); + + // Raw tx is wrapped as a JSON string with the `0x` prefix for Ethereum. + let raw_msg = msgs.iter().find(|m| m.field == "tx-raw").unwrap(); + let raw_entry = parse(&raw_msg.payload); + assert_eq!(raw_entry["value"], "0xdeadbeef"); + } + + #[test] + fn bitcoin_tx_raw_omits_0x_prefix() { + let mut r = row( + DataKind::Transactions, + Some("abc"), + vec![Field::TxRaw(vec![0xde, 0xad, 0xbe, 0xef])], + ); + r.blockchain_type = BlockchainType::Bitcoin; + let msgs = encode_row(&r); + assert_eq!(msgs.len(), 1); + let entry = parse(&msgs[0].payload); + assert_eq!(entry["value"], "deadbeef"); + } + + #[test] + fn trace_row_emits_calls_and_statediff_messages() { + let r = row( + DataKind::TransactionTraces, + Some("0xabc"), + vec![ + Field::Trace(b"{\"t\":1}".to_vec()), + Field::StateDiff(b"{\"s\":1}".to_vec()), + ], + ); + let msgs = encode_row(&r); + assert_eq!(msgs.len(), 2); + let trace = msgs.iter().find(|m| m.field == "trace-calls").unwrap(); + assert_eq!( + trace.properties.get("dedup-key").unwrap(), + "trace-calls:0xblock:tx-0xabc" + ); + let trace_entry = parse(&trace.payload); + assert_eq!(trace_entry["table"], "traces"); + assert_eq!(trace_entry["value"], serde_json::json!({"t": 1})); + let state = msgs.iter().find(|m| m.field == "trace-statediff").unwrap(); + assert_eq!( + state.properties.get("dedup-key").unwrap(), + "trace-statediff:0xblock:tx-0xabc" + ); + let state_entry = parse(&state.payload); + assert_eq!(state_entry["value"], serde_json::json!({"s": 1})); + } + + #[test] + fn uncle_payload_carries_uncle_index() { + let r = row( + DataKind::Blocks, + None, + vec![Field::Uncle { index: 1, json: b"{\"u\":1}".to_vec() }], + ); + let msgs = encode_row(&r); + assert_eq!(msgs.len(), 1); + assert_eq!( + msgs[0].properties.get("dedup-key").unwrap(), + "blocks-uncles:0xblock:uncle-1" + ); + let entry = parse(&msgs[0].payload); + assert_eq!(entry["uncleIndex"], 1); + // Block-kind row → no tx-* fields in the envelope. + assert!(entry.get("txIndex").is_none()); + assert!(entry.get("txId").is_none()); + } + + #[test] + fn partition_key_is_height() { + let r = row( + DataKind::Blocks, + None, + vec![Field::BlockJson(b"{\"h\":1}".to_vec())], + ); + let msgs = encode_row(&r); + assert_eq!(msgs[0].partition_key, "100"); + } + + #[test] + fn timestamp_serializes_as_iso_8601() { + // Real Ethereum block timestamp value (0x689aad27 = 1754967335 s + // since epoch) — matches eth_getBlockByNumber's `timestamp` field + // for block 23110555 on mainnet. + let ts = Utc.timestamp_opt(0x689aad27, 0).unwrap(); + let mut r = row( + DataKind::Blocks, + None, + vec![Field::BlockJson(b"{\"h\":1}".to_vec())], + ); + r.timestamp = ts; + let msgs = encode_row(&r); + let entry = parse(&msgs[0].payload); + // chrono's serde impl uses the `Z` form for UTC... + assert_eq!(entry["timestamp"], "2025-08-12T02:55:35Z"); + // ...whereas `to_rfc3339()` (what we use for the broker header) + // spells the same offset as `+00:00`. Both are valid RFC 3339; the + // test pins the current behaviour so a future codec swap doesn't + // silently change the on-the-wire format. + assert_eq!( + msgs[0].properties.get("timestamp").map(|s| s.as_str()), + Some("2025-08-12T02:55:35+00:00") + ); + } + + /// Visual snapshot: serializing a row whose value is a JSON object (the + /// common case — block JSON, tx JSON, receipts, traces, state diff) + /// must produce the exact envelope shape consumers depend on. Pinning + /// the literal output here makes any accidental field rename, reorder + /// or whitespace drift fail loudly and be reviewable by eye in the diff. + #[test] + fn serializes_object_value_to_expected_json() { + let r = ArchiveRow { + kind: DataKind::Transactions, + blockchain_type: BlockchainType::Ethereum, + blockchain_id: "ETH".to_string(), + archive_ts: Utc.timestamp_opt(0, 0).unwrap(), + height: 23110555, + block_id: "0xbbb".to_string(), + timestamp: Utc.timestamp_opt(0x689aad27, 0).unwrap(), + parent_id: Some("0xparent".to_string()), + tx_index: Some(3), + tx_id: Some("0xaaa".to_string()), + tx_count: Some(25), + fields: vec![Field::TxJson( + br#"{"hash":"0xaaa","nonce":"0x1","input":"0x"}"#.to_vec(), + )], + }; + let msgs = encode_row(&r); + assert_eq!(msgs.len(), 1); + let payload = std::str::from_utf8(&msgs[0].payload).expect("utf-8 json"); + assert_eq!( + payload, + r#"{"blockchain":"ETH","timestamp":"2025-08-12T02:55:35Z","table":"transactions","field":"tx-json","height":23110555,"blockId":"0xbbb","parentId":"0xparent","txIndex":3,"txCount":25,"txId":"0xaaa","value":{"hash":"0xaaa","nonce":"0x1","input":"0x"}}"# + ); + } + + /// Visual snapshot for the raw-tx variant: `value` is a JSON *string* + /// (the chain-prefixed hex), not an object. This pins the envelope's + /// behaviour for non-object values so a future regression that wraps + /// the hex in `{...}` or strips the prefix is caught here. + #[test] + fn serializes_string_value_to_expected_json() { + let r = ArchiveRow { + kind: DataKind::Transactions, + blockchain_type: BlockchainType::Ethereum, + blockchain_id: "ETH".to_string(), + archive_ts: Utc.timestamp_opt(0, 0).unwrap(), + height: 23110555, + block_id: "0xbbb".to_string(), + timestamp: Utc.timestamp_opt(0x689aad27, 0).unwrap(), + parent_id: Some("0xparent".to_string()), + tx_index: Some(3), + tx_id: Some("0xaaa".to_string()), + tx_count: Some(25), + fields: vec![Field::TxRaw(vec![0xde, 0xad, 0xbe, 0xef])], + }; + let msgs = encode_row(&r); + assert_eq!(msgs.len(), 1); + let payload = std::str::from_utf8(&msgs[0].payload).expect("utf-8 json"); + assert_eq!( + payload, + r#"{"blockchain":"ETH","timestamp":"2025-08-12T02:55:35Z","table":"transactions","field":"tx-raw","height":23110555,"blockId":"0xbbb","parentId":"0xparent","txIndex":3,"txCount":25,"txId":"0xaaa","value":"0xdeadbeef"}"# + ); + } + + /// Re-org regression: two blocks at the *same height* with different + /// block ids — even when they share the same tx hash — must produce + /// *different* dedup keys. Otherwise consumers would silently treat the + /// replacement block's txes as duplicates and drop real state. + #[test] + fn dedup_key_differs_across_same_height_reorg() { + let original = ArchiveRow { + kind: DataKind::Transactions, + blockchain_type: BlockchainType::Ethereum, + blockchain_id: "ETH".to_string(), + archive_ts: Utc::now(), + height: 100, + block_id: "0xAAA".to_string(), + timestamp: Utc.timestamp_millis_opt(0).unwrap(), + parent_id: None, + tx_index: Some(0), + tx_id: Some("0xtx".to_string()), + tx_count: None, + fields: vec![Field::TxJson(b"{\"v\":1}".to_vec())], + }; + let reorged = ArchiveRow { + block_id: "0xBBB".to_string(), + fields: vec![Field::TxJson(b"{\"v\":2}".to_vec())], + ..original.clone() + }; + let original_key = encode_row(&original)[0] + .properties + .get("dedup-key") + .cloned() + .unwrap(); + let reorged_key = encode_row(&reorged)[0] + .properties + .get("dedup-key") + .cloned() + .unwrap(); + assert_ne!( + original_key, reorged_key, + "same height + same tx hash on different blocks must not collide" + ); + assert_eq!(original_key, "tx-json:0xAAA:tx-0xtx"); + assert_eq!(reorged_key, "tx-json:0xBBB:tx-0xtx"); + } + + /// `topic_labels_for` filters by blockchain shape: Bitcoin gets no + /// uncles, no receipts, no traces — even if `--tables` includes traces + /// (a misconfig). + #[test] + fn topic_labels_for_bitcoin_excludes_eth_only_topics() { + use crate::archiver::datakind::{BlockOptions, DataOptions, TraceOptions, TxOptions}; + let opts = DataOptions { + overwrite: true, + block: Some(BlockOptions::default()), + tx: Some(TxOptions::default()), + // Even with traces ostensibly enabled, Bitcoin must not get + // trace topics. + trace: Some(TraceOptions::default()), + }; + let labels = topic_labels_for(BlockchainType::Bitcoin, &opts); + assert_eq!(labels, vec!["blocks", "tx-json", "tx-raw"]); + } + + /// `topic_labels_for` defaults for Ethereum + blocks/txes: receipts and + /// uncles ARE included; trace topics are NOT (trace = None). + #[test] + fn topic_labels_for_ethereum_default_tables_excludes_traces() { + use crate::archiver::datakind::{BlockOptions, DataOptions, TxOptions}; + let opts = DataOptions { + overwrite: true, + block: Some(BlockOptions::default()), + tx: Some(TxOptions::default()), + trace: None, + }; + let labels = topic_labels_for(BlockchainType::Ethereum, &opts); + assert_eq!( + labels, + vec!["blocks", "blocks-uncles", "tx-json", "tx-raw", "tx-receipts"] + ); + } + + /// Ethereum with traces enabled: both trace topics surface when + /// `TraceOptions` requests both fields. + #[test] + fn topic_labels_for_ethereum_with_full_traces() { + use crate::archiver::datakind::{BlockOptions, DataOptions, TraceOptions, TxOptions}; + let opts = DataOptions { + overwrite: true, + block: Some(BlockOptions::default()), + tx: Some(TxOptions::default()), + trace: Some(TraceOptions { + include_trace: true, + include_state_diff: true, + }), + }; + let labels = topic_labels_for(BlockchainType::Ethereum, &opts); + assert_eq!( + labels, + vec![ + "blocks", + "blocks-uncles", + "tx-json", + "tx-raw", + "tx-receipts", + "trace-calls", + "trace-statediff", + ] + ); + } + + /// Ethereum with `--fields.trace calls`: only the calls trace topic + /// surfaces; statediff is suppressed. + #[test] + fn topic_labels_for_ethereum_trace_calls_only() { + use crate::archiver::datakind::{BlockOptions, DataOptions, TraceOptions, TxOptions}; + let opts = DataOptions { + overwrite: true, + block: Some(BlockOptions::default()), + tx: Some(TxOptions::default()), + trace: Some(TraceOptions { + include_trace: true, + include_state_diff: false, + }), + }; + let labels = topic_labels_for(BlockchainType::Ethereum, &opts); + assert!(labels.contains(&"trace-calls")); + assert!(!labels.contains(&"trace-statediff")); + } + + /// `--tables blocks` only: tx-* and trace-* topics must all be absent. + #[test] + fn topic_labels_for_blocks_only_returns_just_block_topics() { + use crate::archiver::datakind::{BlockOptions, DataOptions}; + let opts = DataOptions { + overwrite: true, + block: Some(BlockOptions::default()), + tx: None, + trace: None, + }; + let eth = topic_labels_for(BlockchainType::Ethereum, &opts); + assert_eq!(eth, vec!["blocks", "blocks-uncles"]); + let btc = topic_labels_for(BlockchainType::Bitcoin, &opts); + assert_eq!(btc, vec!["blocks"]); + } + + /// Sanity: every label produced by [`encode_field`] for any [`Field`] + /// variant must appear in the maximal [`topic_labels_for`] output — + /// otherwise the storage layer wouldn't have pre-created a producer for + /// it and `append` would fail. Uses Ethereum + all-options as the + /// superset since that's the largest possible producer set. + #[test] + fn every_encoded_field_has_a_corresponding_topic() { + // One representative row for each publishable Field variant. + let r = ArchiveRow { + kind: DataKind::Transactions, + blockchain_type: BlockchainType::Ethereum, + blockchain_id: "ETH".to_string(), + archive_ts: Utc::now(), + height: 1, + block_id: "0x".to_string(), + timestamp: Utc.timestamp_millis_opt(0).unwrap(), + parent_id: None, + tx_index: Some(0), + tx_id: Some("0xabc".to_string()), + tx_count: None, + fields: vec![ + Field::BlockJson(b"{}".to_vec()), + Field::Uncle { index: 0, json: b"{}".to_vec() }, + Field::TxJson(b"{}".to_vec()), + Field::TxRaw(vec![]), + Field::Receipt(b"{}".to_vec()), + Field::From("from".to_string()), + Field::To("to".to_string()), + Field::Trace(b"{}".to_vec()), + Field::StateDiff(b"{}".to_vec()), + ], + }; + use crate::archiver::datakind::{BlockOptions, DataOptions, TraceOptions, TxOptions}; + let max_options = DataOptions { + overwrite: true, + block: Some(BlockOptions::default()), + tx: Some(TxOptions::default()), + trace: Some(TraceOptions { + include_trace: true, + include_state_diff: true, + }), + }; + let produced: HashSet<&'static str> = + encode_row(&r).into_iter().map(|m| m.field).collect(); + let declared: HashSet<&'static str> = + topic_labels_for(BlockchainType::Ethereum, &max_options) + .into_iter() + .collect(); + // The two sides must match exactly: any label encode_row produces + // for the full Ethereum row must be pre-created by Pulsar, and we + // shouldn't be pre-creating dead topics either. + assert_eq!(produced, declared); + } +} diff --git a/src/global.rs b/src/global.rs index dff905c..daa2533 100644 --- a/src/global.rs +++ b/src/global.rs @@ -1,7 +1,9 @@ use std::sync::Mutex; +use std::time::Duration; use apache_avro::{Codec, ZstandardSettings}; use lazy_static::lazy_static; -use crate::args::{Args, Compression}; +use tokio_retry2::strategy::{jitter, ExponentialFactorBackoff}; +use crate::args::{Args, Compression, RetryMode}; /// Configuration for parallelism limits across different archival operations. /// @@ -25,12 +27,26 @@ lazy_static! { static ref COMPRESSION: Mutex = Mutex::new(Compression::Zstd); static ref DRY_RUN: Mutex = Mutex::new(false); static ref THREADS: Mutex = Mutex::new(ThreadsConfig { api: 16, tx: 8, trace: 4, blocks: 8 }); + // Default to the bounded policy (matching pre-flag behaviour) until + // `set_retry_policy` resolves the real value at startup. + static ref RETRY_POLICY: Mutex = Mutex::new(RetryPolicy::Bounded { + max_attempts: DEFAULT_RETRY_MAX_ATTEMPTS, + }); } pub fn get_shutdown() -> shutdown::Shutdown { SHUTDOWN.clone() } +/// Avro codec for `--format=avro` writes. +/// +/// Zstd level **9** is intentional: Avro files are long-lived archive +/// artifacts (held for months/years, scanned by downstream batch jobs), so we +/// pay one-time CPU at write time in exchange for the smaller storage +/// footprint that compounds across the whole archive. The level is high +/// enough to noticeably beat default (~level 3) on the JSON-heavy payloads +/// dshackle-archive writes, while still well below the diminishing-returns +/// zone above ~15. pub fn get_avro_codec() -> Codec { let compression = COMPRESSION.lock().unwrap(); match *compression { @@ -39,6 +55,36 @@ pub fn get_avro_codec() -> Codec { } } +/// Map the user-selected compression to a Pulsar producer compression option. +/// +/// Honours the same `--compression` flag the Avro path uses, so a single +/// archive run uses a consistent codec choice across whichever target it +/// writes to. Pulsar's `compression` feature is enabled by default in the +/// upstream crate, so both `Zstd` and `Snappy` are always available here. +/// +/// Note the **level asymmetry vs. [`get_avro_codec`]**: this returns +/// `CompressionZstd::default()` (≈ level 3), whereas the Avro path uses +/// level 9. The trade-off is intentional: +/// +/// - Avro files are long-lived archive artifacts where write CPU amortizes +/// across years of cold storage — level 9 favours ratio. +/// - Broker topics are typically short-retention live streams. Producer-side +/// compression sits on the latency path of every published message, so a +/// faster, ratio-modest codec is the better default. If someone needs +/// tighter compression on a Pulsar topic they can negotiate it +/// broker-side; for our v1 we keep write latency low. +pub fn get_pulsar_compression() -> pulsar::compression::Compression { + let compression = COMPRESSION.lock().unwrap(); + match *compression { + Compression::Snappy => pulsar::compression::Compression::Snappy( + pulsar::compression::CompressionSnappy::default(), + ), + Compression::Zstd => pulsar::compression::Compression::Zstd( + pulsar::compression::CompressionZstd::default(), + ), + } +} + pub fn set_compression(args: &Args) { let compression = args.compression.clone().unwrap_or(Compression::Zstd); let mut comp = COMPRESSION.lock().unwrap(); @@ -104,3 +150,126 @@ pub fn get_threads() -> ThreadsConfig { fn read_env(name: &str) -> Option { std::env::var(name).ok().and_then(|v| v.parse().ok()) } + +/// Number of attempts the `Bounded` retry policy uses. Matches the +/// hardcoded `.take(10)` the per-RPC helpers used to carry inline before this +/// became a global. Not exposed as a CLI knob yet — most callers either +/// accept the default or switch to `Forever`. +pub const DEFAULT_RETRY_MAX_ATTEMPTS: usize = 10; + +/// Runtime retry policy resolved from CLI args (and the target type). +/// +/// File targets default to [`RetryPolicy::Bounded`]: a transient node failure +/// leaves a gap that the `fix`/`verify` commands can repair later. Ordered +/// streaming targets default to [`RetryPolicy::Forever`]: a missing record +/// permanently breaks the topic-order contract, so the writer must wait the +/// node out instead. The user can override either default via `--retry`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RetryPolicy { + Bounded { max_attempts: usize }, + Forever, +} + +/// Initialise the retry policy from CLI args. +/// +/// Explicit `--retry` wins; otherwise the default is derived from the target +/// type — streaming-ordered targets need [`RetryPolicy::Forever`] to keep +/// their order contract; everything else can fail fast and be repaired +/// later. Logs the resolved policy so operators can see what's in effect. +pub fn set_retry_policy(args: &Args) { + let policy = resolve_retry_policy(args); + tracing::info!("Retry policy: {:?}", policy); + *RETRY_POLICY.lock().unwrap() = policy; +} + +fn resolve_retry_policy(args: &Args) -> RetryPolicy { + match args.retry { + Some(RetryMode::Bounded) => RetryPolicy::Bounded { + max_attempts: DEFAULT_RETRY_MAX_ATTEMPTS, + }, + Some(RetryMode::Forever) => RetryPolicy::Forever, + None => { + if crate::storage::is_pulsar(args) { + RetryPolicy::Forever + } else { + RetryPolicy::Bounded { + max_attempts: DEFAULT_RETRY_MAX_ATTEMPTS, + } + } + } + } +} + +/// Current retry policy. Cheap (one mutex lock); callers can call it once +/// per retry-strategy build. +pub fn get_retry_policy() -> RetryPolicy { + *RETRY_POLICY.lock().unwrap() +} + +/// Build the iterator passed to `tokio_retry2::Retry::spawn` for a single +/// fetch attempt sequence. +/// +/// Same shape regardless of policy — exponential backoff with jitter, capped +/// at `max_delay_secs` between attempts — but the iterator is bounded or +/// unbounded based on [`get_retry_policy`]. Returned boxed so both branches +/// have the same type at the call site (the underlying iterator types +/// otherwise differ between `Take<…>` and the unbounded form). +pub fn retry_strategy(max_delay_secs: u64) -> Box + Send> { + let base = ExponentialFactorBackoff::from_millis(100, 1.75) + .max_delay(Duration::from_secs(max_delay_secs)) + .map(jitter); + match get_retry_policy() { + RetryPolicy::Bounded { max_attempts } => Box::new(base.take(max_attempts)), + RetryPolicy::Forever => Box::new(base), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn args_with(retry: Option, pulsar: bool) -> Args { + let stream = if pulsar { + Some(crate::args::Stream { + stream_url: Some("pulsar://localhost:6650".to_string()), + stream_topics: Some("persistent://public/default/x".to_string()), + }) + } else { + None + }; + Args { + retry, + stream, + ..Args::default() + } + } + + #[test] + fn explicit_bounded_wins_over_pulsar_default() { + let policy = resolve_retry_policy(&args_with(Some(RetryMode::Bounded), true)); + assert!(matches!(policy, RetryPolicy::Bounded { .. })); + } + + #[test] + fn explicit_forever_wins_over_file_default() { + let policy = resolve_retry_policy(&args_with(Some(RetryMode::Forever), false)); + assert!(matches!(policy, RetryPolicy::Forever)); + } + + #[test] + fn pulsar_defaults_to_forever() { + let policy = resolve_retry_policy(&args_with(None, true)); + assert!(matches!(policy, RetryPolicy::Forever)); + } + + #[test] + fn non_pulsar_defaults_to_bounded() { + let policy = resolve_retry_policy(&args_with(None, false)); + assert!(matches!( + policy, + RetryPolicy::Bounded { + max_attempts: DEFAULT_RETRY_MAX_ATTEMPTS + } + )); + } +} diff --git a/src/main.rs b/src/main.rs index 1b83d18..b772f83 100644 --- a/src/main.rs +++ b/src/main.rs @@ -83,6 +83,7 @@ async fn main_inner() -> Result<()> { global::set_dry_run(&args); global::set_compression(&args); global::set_threads(&args); + global::set_retry_policy(&args); progress::start(); if let Some(ref addr) = args.metrics { @@ -107,6 +108,20 @@ async fn main_inner() -> Result<()> { } } + if storage::is_pulsar(&args) { + if args.command != Command::Stream { + return Err(anyhow!( + "{:?} is not supported with the Pulsar streaming target (topics are append-only — only `stream` can publish to them)", + args.command + )); + } + if args.continue_last { + return Err(anyhow!( + "--continue is not supported by the Pulsar streaming target (no tail-scan capability in v1)" + )); + } + } + let chain_ref = ChainRef::from_str(&args.blockchain) .map_err(|_| anyhow!("Unsupported blockchain: {}", args.blockchain))?; let chain_type = BlockchainType::try_from(chain_ref) @@ -126,7 +141,19 @@ async fn main_inner() -> Result<()> { } async fn run(builder: Builder, args: &Args) -> Result<()> { - if storage::is_fs(&args) { + if let Some(stream) = args.stream.as_ref() { + if let Some(url) = stream.stream_url.as_deref() { + if !storage::is_pulsar(args) { + return Err(anyhow!( + "Unsupported --stream.url scheme: {} (only pulsar:// is supported today)", + url + )); + } + } + } + if storage::is_pulsar(&args) { + run_with_write_target(builder, storage::create_pulsar::(&args).await?, args).await + } else if storage::is_fs(&args) { match args.format { Format::Avro => run_with_read_target(builder, storage::create_fs(&args)?, args).await, Format::Json => run_with_scan_target(builder, storage::create_fs_json(&args)?, args).await, @@ -182,6 +209,26 @@ async fn run_with_scan_target( + builder: Builder, + target: TS, + args: &Args, +) -> Result<()> { + let builder = build_with_target(builder, target, args).await?; + match args.command { + Command::Stream => builder.stream_write_only(args).await.execute().await, + _ => Err(anyhow!( + "{:?} is not supported by a write-only streaming target", + args.command + )), + } +} + async fn build_with_target( builder: Builder, target: TS, @@ -246,12 +293,24 @@ impl BuilderWithTarget where B: BlockchainTypes, TS: WriteTarget { impl BuilderWithData where B: BlockchainTypes + 'static, TS: WriteTarget + 'static { - /// `stream` reads existing data to honour `--continue`, so it requires - /// [`ScanTarget`]. + /// `stream` with `--continue` support — requires [`ScanTarget`] so it can + /// enumerate already-archived data before starting the live tail. async fn stream(self, args: &Args) -> StreamCommand where TS: ScanTarget, { + let notifier = self.parent.parent.notifier.unwrap(); + let notifications = notifier.start(); + let archiver = Archiver::new( + Arc::new(self.parent.target), Arc::new(self.data), notifications + ); + let command = StreamCommand::new_with_resume(&args, archiver).await.unwrap(); + command + } + + /// `stream` against a write-only target (Pulsar). `--continue` is rejected + /// because there's no scan capability to compute a resume point from. + async fn stream_write_only(self, args: &Args) -> StreamCommand { let notifier = self.parent.parent.notifier.unwrap(); let notifications = notifier.start(); let archiver = Archiver::new( diff --git a/src/metrics/archive.rs b/src/metrics/archive.rs index 5fcbb35..29fc03a 100644 --- a/src/metrics/archive.rs +++ b/src/metrics/archive.rs @@ -66,14 +66,14 @@ impl ArchiveMetrics { /// Record that `n` items of the given data kind have been processed. pub fn add_items(&self, kind: &DataKind, direction: &Direction, n: usize) { self.items - .with_label_values(&[kind.metrics_label(), direction.metrics_label()]) + .with_label_values(&[kind.label(), direction.metrics_label()]) .inc_by(n as f64); } /// Record that `n` bytes of the given data kind have been transferred. pub fn add_bytes(&self, kind: &DataKind, direction: &Direction, n: usize) { self.bytes - .with_label_values(&[kind.metrics_label(), direction.metrics_label()]) + .with_label_values(&[kind.label(), direction.metrics_label()]) .inc_by(n as f64); } diff --git a/src/record.rs b/src/record.rs index 63b0096..81aee47 100644 --- a/src/record.rs +++ b/src/record.rs @@ -61,12 +61,17 @@ pub struct ArchiveRow { /// Block timestamp as reported by the blockchain node. pub timestamp: DateTime, - /// Block-kind only: parent block hash. + /// Parent block hash. pub parent_id: Option, /// Tx/Trace-kind only: transaction index within the block. pub tx_index: Option, /// Tx/Trace-kind only: transaction id (hash). pub tx_id: Option, + /// Total number of transactions in the enclosing block. Lets a + /// consumer reading a single tx/trace message know its position (tx + /// `N` of `tx_count`); also surfaces the block's tx volume on block + /// rows. + pub tx_count: Option, pub fields: Vec, } @@ -106,9 +111,11 @@ pub enum Field { TxRaw(Vec), /// Ethereum-only: the transaction receipt JSON. Receipt(Vec), - /// Ethereum-only: convenience field carrying the `from` address. + /// Ethereum-only: `from` address as a dedicated column for table + /// formats (Avro). Streaming and per-field JSON skip it — already in + /// the tx JSON. From(String), - /// Ethereum-only: convenience field carrying the `to` address. + /// Ethereum-only: `to` address. Same usage as [`Field::From`]. To(String), // ---- Trace-kind fields ---- @@ -117,3 +124,46 @@ pub enum Field { /// `debug_traceTransaction` with `prestateTracer`. StateDiff(Vec), } + +impl Field { + /// Singular content-type identifier — the kind of value this variant + /// carries, independent of any table context. + /// + /// Suitable for callers where the enclosing table is already known + /// (e.g. directory-based layouts where the table appears in the path): + /// the name doesn't need to repeat it. Plural is reserved for variants + /// whose payload is itself a collection (`calls` — the callTracer + /// returns a nested call tree). + pub fn name(&self) -> &'static str { + match self { + Field::BlockJson(_) => "block", + Field::Uncle { .. } => "uncle", + Field::TxJson(_) => "tx", + Field::TxRaw(_) => "raw", + Field::Receipt(_) => "receipt", + Field::From(_) => "from", + Field::To(_) => "to", + Field::Trace(_) => "calls", + Field::StateDiff(_) => "statediff", + } + } + + /// Streaming topic suffix for this variant. Used by + /// [`crate::formats::stream`] as the per-field topic label + /// (`-`), where topics share a flat namespace and + /// the table context isn't otherwise carried. Keep the values stable + /// — they're consumer-visible. + pub fn topic_label(&self) -> &'static str { + match self { + Field::BlockJson(_) => "blocks", + Field::Uncle { .. } => "blocks-uncles", + Field::TxJson(_) => "tx-json", + Field::TxRaw(_) => "tx-raw", + Field::Receipt(_) => "tx-receipts", + Field::From(_) => "tx-from", + Field::To(_) => "tx-to", + Field::Trace(_) => "trace-calls", + Field::StateDiff(_) => "trace-statediff", + } + } +} diff --git a/src/storage/json_fs.rs b/src/storage/json_fs.rs index c2da34f..5f0cd23 100644 --- a/src/storage/json_fs.rs +++ b/src/storage/json_fs.rs @@ -220,6 +220,7 @@ mod tests { parent_id: Some(format!("0xparent{}", height - 1)), tx_index: None, tx_id: None, + tx_count: None, fields: vec![ Field::BlockJson(format!("{{\"h\":{}}}", height).into_bytes()), Field::Uncle { index: 0, json: b"U0".to_vec() }, @@ -239,6 +240,7 @@ mod tests { parent_id: None, tx_index: Some(0), tx_id: Some(tx_id.to_string()), + tx_count: None, fields: vec![ Field::TxJson(b"{}".to_vec()), Field::TxRaw(vec![0xde, 0xad, 0xbe, 0xef]), diff --git a/src/storage/json_objects.rs b/src/storage/json_objects.rs index 666a926..0333dbb 100644 --- a/src/storage/json_objects.rs +++ b/src/storage/json_objects.rs @@ -182,6 +182,7 @@ mod tests { parent_id: Some(format!("0xparent{}", height.saturating_sub(1))), tx_index: None, tx_id: None, + tx_count: None, fields: vec![Field::BlockJson(format!("{{\"h\":{}}}", height).into_bytes())], } } @@ -198,6 +199,7 @@ mod tests { parent_id: None, tx_index: Some(0), tx_id: Some(tx_id.to_string()), + tx_count: None, fields: vec![ Field::TxJson(b"{}".to_vec()), Field::TxRaw(vec![0xde, 0xad, 0xbe, 0xef]), diff --git a/src/storage/mod.rs b/src/storage/mod.rs index e6bf078..0e1071d 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -33,6 +33,7 @@ pub mod fs; pub mod json_fs; pub mod json_objects; pub mod objects; +pub mod pulsar; mod avro_reader; mod copy; mod sorted_files; @@ -45,6 +46,55 @@ pub fn is_fs(args: &Args) -> bool { args.dir.is_some() && !is_s3(args) } +/// True when the user selected a streaming target whose URL is a Pulsar URL +/// (`pulsar://…`). Mutually exclusive with the file targets: when this is true +/// the storage layer ignores `--dir` and `--auth.aws.*` and routes records to +/// per-field topics instead. +pub fn is_pulsar(args: &Args) -> bool { + args.stream + .as_ref() + .map(|s| s.is_pulsar()) + .unwrap_or(false) +} + +/// Build a [`pulsar::PulsarStorage`] from the user-supplied `--stream.*` args. +/// +/// The producer set is restricted to topics that make sense for the running +/// blockchain (Bitcoin omits receipts / uncles / traces) AND for the +/// caller's `--tables` / `--fields.trace` selection (traces topics are +/// skipped unless `traces` is in `--tables`, and within traces each +/// sub-topic is gated on its own field flag). See +/// [`crate::formats::stream::topic_labels_for`] for the exact mapping. +/// +/// `--stream.topics` is taken verbatim — callers are expected to include +/// the Pulsar topic path up to and including the blockchain segment (e.g. +/// `persistent://public/default/archive-eth`). +pub async fn create_pulsar( + value: &Args, +) -> Result { + let stream = value + .stream + .as_ref() + .ok_or_else(|| anyhow!("No --stream.* options set"))?; + let url = stream + .stream_url + .clone() + .ok_or_else(|| anyhow!("--stream.url is required for a streaming target"))?; + let prefix = stream + .stream_topics + .clone() + .ok_or_else(|| anyhow!("--stream.topics is required for a streaming target"))?; + let data_options = DataOptions::from(value); + let labels = crate::formats::stream::topic_labels_for(B::BLOCKCHAIN_TYPE, &data_options); + tracing::info!( + "Using Pulsar streaming target at {} with topic prefix {} ({} topics)", + url, + prefix, + labels.len() + ); + pulsar::PulsarStorage::new(url, prefix, &labels).await +} + /// /// Compute the S3 key prefix (without bucket) under which the archive lives. /// @@ -193,6 +243,21 @@ pub trait WriteTarget: Send + Sync { /// /// Returns `None` when `overwrite` is false and the destination already exists. async fn create(&self, kind: DataKind, range: &Range, overwrite: bool) -> Result>; + + /// + /// Whether the archiver must hand records to the writer in strict + /// chain-natural order (block height for blocks; flat + /// `(block_position, tx_index)` ordinal for txes/traces). + /// + /// Streaming backends (Pulsar, future Kafka) must override this to `true` + /// because brokers preserve messages in *publish* order, so re-ordering + /// across parallel fetches would corrupt the consumer's view. File + /// backends (Avro, JSON) leave this `false`: files accept records in any + /// order, and bypassing the ordering layer avoids a per-row channel hop + /// for batch archives that may push hundreds of thousands of rows. + fn needs_ordering(&self) -> bool { + false + } } /// diff --git a/src/storage/objects.rs b/src/storage/objects.rs index 551c718..9a5577b 100644 --- a/src/storage/objects.rs +++ b/src/storage/objects.rs @@ -435,6 +435,7 @@ mod tests { parent_id: Some("0xdb10afd3efa45327eb284c83cc925bd9bd7966aea53067c1eebe0724d124ec1e".to_string()), tx_index: None, tx_id: None, + tx_count: None, fields: vec![Field::BlockJson(vec![1, 2, 3])], } } diff --git a/src/storage/pulsar.rs b/src/storage/pulsar.rs new file mode 100644 index 0000000..0d001f2 --- /dev/null +++ b/src/storage/pulsar.rs @@ -0,0 +1,404 @@ +// Copyright 2026 EmeraldPay Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. + +//! Apache Pulsar streaming target. +//! +//! Implements [`WriteTarget`] only — topics are append-only logs, so +//! `archive`, `fix`, `verify`, and `compact` are rejected upfront in +//! [`crate::main`]. +//! +//! ## Topic layout +//! +//! One topic per field, named `-`. The label set comes +//! from [`crate::formats::stream::topic_labels_for`], so only topics +//! relevant to the running blockchain and the user's `--tables` / +//! `--fields.trace` selection are created. Producers are built eagerly +//! so the first append doesn't pay broker-side latency. +//! +//! ## Ordering & re-org handling +//! +//! Pulsar preserves per-partition publish order *as long as the producer +//! submits messages in order*. A per-producer [`tokio::sync::Mutex`] +//! serializes the send-and-await-ack step for each topic. Messages are +//! keyed by block height, so every field of one block — and any +//! same-height re-org replacement — lands on the same partition in +//! publish order. +//! +//! When a re-org invalidates a block mid-publish, the archiver's cancel +//! path abandons the in-flight ordered sink (see +//! [`crate::archiver::order::OrderedSink::abandon`]); the replacement +//! block then re-publishes with a fresh [`dedup-key`] property so +//! consumers can collapse superseded messages. Transient node failures +//! that would otherwise leave a gap are absorbed by +//! [`crate::global::RetryPolicy::Forever`] (the default for Pulsar +//! targets), which retries until either the data arrives or the re-org +//! token cancels the block. +//! +//! [`dedup-key`]: crate::formats::stream + +use std::collections::HashMap; +use std::sync::Arc; + +use anyhow::{anyhow, Result}; +use async_trait::async_trait; +use pulsar::producer::{Producer, ProducerOptions}; +use pulsar::{Pulsar, TokioExecutor}; +use tokio::sync::Mutex; + +use crate::archiver::datakind::DataKind; +use crate::archiver::range::Range; +use crate::formats::stream; +use crate::record::ArchiveRow; +use crate::storage::{TargetFile, TargetFileWriter, WriteTarget}; + +/// Apache Pulsar streaming target. See the module-level doc for the +/// topic layout and ordering contract. +pub struct PulsarStorage { + topic_prefix: String, + /// Pre-created producers keyed by topic label. Each producer owns + /// the broker connection internally, so no separate [`Pulsar`] + /// client handle is needed. + producers: Arc>>>>, +} + +impl PulsarStorage { + /// Connect to the broker and pre-create one producer per topic in + /// `labels` (named `-