diff --git a/crates/autopilot/src/domain/mod.rs b/crates/autopilot/src/domain/mod.rs index ed1eb17793..7f6fec33fa 100644 --- a/crates/autopilot/src/domain/mod.rs +++ b/crates/autopilot/src/domain/mod.rs @@ -21,10 +21,28 @@ pub struct Metrics { /// Tracks settlements that couldn't be matched to the database solutions. #[metric(labels("solver_address"))] pub inconsistent_settlements: prometheus::IntCounterVec, + + /// Tracks trades whose surplus, fee or fee breakdown calculation failed + /// and fell back to zeroed values. + #[metric(labels("kind"))] + pub settlement_math_errors: prometheus::IntCounterVec, } impl Metrics { fn get() -> &'static Self { Metrics::instance(observe::metrics::get_storage_registry()).unwrap() } + + /// Publishes every `settlement_math_errors` series at zero. A counter that + /// only appears with the first failure is born reading one, leaving + /// `increase()` nothing to subtract from, so the alert would miss that + /// first failure and only catch the second one. + pub(crate) fn init_settlement_math_errors() { + for kind in ["surplus", "fee", "fee_breakdown"] { + Self::get() + .settlement_math_errors + .with_label_values(&[kind]) + .reset(); + } + } } diff --git a/crates/autopilot/src/domain/settlement/mod.rs b/crates/autopilot/src/domain/settlement/mod.rs index 711aa5b16e..1b2eb7d44a 100644 --- a/crates/autopilot/src/domain/settlement/mod.rs +++ b/crates/autopilot/src/domain/settlement/mod.rs @@ -106,6 +106,10 @@ impl Settlement { trade = %trade.uid(), "possible incomplete surplus calculation", ); + Metrics::get() + .settlement_math_errors + .with_label_values(&["surplus"]) + .inc(); num::zero() }); surplus = surplus + trade_surplus; @@ -118,6 +122,10 @@ impl Settlement { trade = %trade.uid(), "possible incomplete fee calculation", ); + Metrics::get() + .settlement_math_errors + .with_label_values(&["fee"]) + .inc(); num::zero() }); fee = fee + trade_fee; @@ -128,6 +136,10 @@ impl Settlement { trade = %trade.uid(), "possible incomplete fee breakdown calculation", ); + Metrics::get() + .settlement_math_errors + .with_label_values(&["fee_breakdown"]) + .inc(); trade::FeeBreakdown { total: eth::Asset { token: trade.sell_token(), diff --git a/crates/autopilot/src/domain/settlement/observer.rs b/crates/autopilot/src/domain/settlement/observer.rs index ec4d758409..66743eb23d 100644 --- a/crates/autopilot/src/domain/settlement/observer.rs +++ b/crates/autopilot/src/domain/settlement/observer.rs @@ -12,7 +12,10 @@ use { crate::{ - domain::settlement::{self, Settlement}, + domain::{ + Metrics, + settlement::{self, Settlement}, + }, infra, }, anyhow::{Context, Result, anyhow}, @@ -31,6 +34,7 @@ impl Observer { /// Creates a new Observer and asynchronously schedules the first update /// run. pub fn new(eth: infra::Ethereum, persistence: infra::Persistence) -> Self { + Metrics::init_settlement_math_errors(); Self { eth, persistence } }