Add a metric for settlement fee and surplus calculation failures

crates/autopilot/src/domain/mod.rs

@@ -21,10 +21,28 @@ pub struct Metrics {
     /// Tracks settlements that couldn't be matched to the database solutions.
     #[metric(labels("solver_address"))]
     pub inconsistent_settlements: prometheus::IntCounterVec,
+
+    /// Tracks trades whose surplus, fee or fee breakdown calculation failed
+    /// and fell back to zeroed values.
+    #[metric(labels("kind"))]
+    pub settlement_math_errors: prometheus::IntCounterVec,
 }
 
 impl Metrics {
     fn get() -> &'static Self {
         Metrics::instance(observe::metrics::get_storage_registry()).unwrap()
     }
+
+    /// Publishes every `settlement_math_errors` series at zero. A counter that
+    /// only appears with the first failure is born reading one, leaving
+    /// `increase()` nothing to subtract from, so the alert would miss that
+    /// first failure and only catch the second one.
+    pub(crate) fn init_settlement_math_errors() {
+        for kind in ["surplus", "fee", "fee_breakdown"] {
+            Self::get()
+                .settlement_math_errors
+                .with_label_values(&[kind])
+                .reset();
+        }
+    }
 }

crates/autopilot/src/domain/settlement/mod.rs

@@ -106,6 +106,10 @@ impl Settlement {
                             trade = %trade.uid(),
                             "possible incomplete surplus calculation",
                         );
+                        Metrics::get()
+                            .settlement_math_errors
+                            .with_label_values(&["surplus"])
+                            .inc();
                         num::zero()
                     });
             surplus = surplus + trade_surplus;
@@ -118,6 +122,10 @@ impl Settlement {
                         trade = %trade.uid(),
                         "possible incomplete fee calculation",
                     );
+                    Metrics::get()
+                        .settlement_math_errors
+                        .with_label_values(&["fee"])
+                        .inc();
                     num::zero()
                 });
             fee = fee + trade_fee;
@@ -128,6 +136,10 @@ impl Settlement {
                     trade = %trade.uid(),
                     "possible incomplete fee breakdown calculation",
                 );
+                Metrics::get()
+                    .settlement_math_errors
+                    .with_label_values(&["fee_breakdown"])
+                    .inc();
                 trade::FeeBreakdown {
                     total: eth::Asset {
                         token: trade.sell_token(),

crates/autopilot/src/domain/settlement/observer.rs

@@ -12,7 +12,10 @@
 
 use {
     crate::{
-        domain::settlement::{self, Settlement},
+        domain::{
+            Metrics,
+            settlement::{self, Settlement},
+        },
         infra,
     },
     anyhow::{Context, Result, anyhow},
@@ -31,6 +34,7 @@ impl Observer {
     /// Creates a new Observer and asynchronously schedules the first update
     /// run.
     pub fn new(eth: infra::Ethereum, persistence: infra::Persistence) -> Self {
+        Metrics::init_settlement_math_errors();
         Self { eth, persistence }
     }