starknet_transaction_prover: HTTP request count + latency + in-flight metrics

crates/starknet_transaction_prover/README.md

@@ -300,6 +300,9 @@ scrapes bypass CORS and JSON-RPC parsing, and the endpoint is unauthenticated.
 | Metric | Type | Labels | Description |
 |---|---|---|---|
 | `prover_build_info` | gauge | `version`, `git_sha` | Always 1. Identifies the running build from a scrape. `git_sha` comes from the `GIT_SHA` docker build arg, and is `unknown` when the build doesn't pass one. |
+| `prover_http_requests_total` | counter | `method`, `status` | HTTP request count. `method` is a bounded enum (`GET`/`POST`/`PUT`/`DELETE`/`HEAD`/`OPTIONS`/`PATCH`/`other`); `status` is the HTTP status class (`1xx` through `5xx`, plus `other`, and `error` when the tower stack produced no response). Excludes `/health` and `/metrics` probes. |
+| `prover_http_request_duration_seconds` | histogram | `method`, `status` | End-to-end HTTP request latency, using the same bounded `method` and `status` label values as the request counter. Excludes `/health` and `/metrics` probes. |
+| `prover_http_inflight_requests` | gauge | none | Current count of HTTP requests being handled. Decremented via RAII so panics and cancellations don't leak. |
 | `prover_prove_transaction_outcome_total` | counter | `outcome` | Every proving request that reaches the prover, so its total is the shared denominator for all proving rates. `outcome` is a bounded enum: `success`, `failure_validation`, `failure_blocked`, `failure_runner`, `failure_output_parse`, `failure_proving`. |
 | `prover_prove_transaction_duration_seconds` | histogram | `outcome` | Duration of the whole proving call, covering input validation, the optional blocking check, the virtual OS run and proving. Recorded for failures too, so filter on `outcome` for success-only percentiles. |
 | `prover_os_run_duration_seconds` | histogram | none | Virtual OS execution time, recorded for successful runs only. |

crates/starknet_transaction_prover/src/server.rs

@@ -43,6 +43,8 @@ pub const OHTTP_JSONRPSEE_BODY_BUILDER: fn(Full<Bytes>) -> HttpBody = HttpBody::
 /// - `RequestLogLayer` is outermost so the latency it measures covers every other layer.
 /// - `HealthLayer` and `MetricsLayer` sit inside it so probes and scrapes complete before
 ///   CORS/OHTTP.
+/// - `HttpMetricsLayer` records per-request latency. It sits below `HealthLayer` and
+///   `MetricsLayer`, so the probes and scrapes they short-circuit stay out of the distribution.
 /// - `OhttpLayer` must sit OUTSIDE `CompressionLayer` so compression applies to the inner JSON-RPC
 ///   response (the client's inner `Accept-Encoding` travels through BHTTP into jsonrpsee) rather
 ///   than to the OHTTP ciphertext envelope. `MapRequestBodyLayer`/`MapResponseBodyLayer` keep
@@ -56,6 +58,7 @@ macro_rules! prover_http_middleware {
             .layer(RequestLogLayer)
             .layer(HealthLayer)
             .layer($metrics_layer)
+            .layer(HttpMetricsLayer)
             .option_layer($cors_layer)
             .layer(MapRequestBodyLayer::new(HttpBody::new))
             .option_layer($ohttp_layer)
@@ -69,6 +72,7 @@ pub mod config;
 pub mod cors;
 pub mod errors;
 pub mod health;
+pub mod http_metrics;
 pub mod log_redact;
 pub mod metrics;
 #[cfg(test)]
@@ -86,6 +90,7 @@ pub mod test_recorder;
 pub mod tls;
 
 pub use health::{HealthLayer, HEALTH_PATH};
+pub use http_metrics::HttpMetricsLayer;
 pub use metrics::{MetricsLayer, METRICS_PATH};
 pub use request_log::{RequestLogLayer, REQUEST_ID_HEADER};
 pub use request_span::RequestSpanLayer;

crates/starknet_transaction_prover/src/server/http_metrics.rs

@@ -0,0 +1,138 @@
+//! tower middleware that records HTTP-level Prometheus metrics:
+//! request count, latency histogram, and an RAII-guarded in-flight gauge.
+
+use std::task::{Context, Poll};
+use std::time::Instant;
+
+use http::{Method, Request, Response, StatusCode};
+use jsonrpsee::server::HttpBody;
+use tower::{Layer, Service};
+
+use crate::server::metrics::GaugeGuard;
+
+#[cfg(test)]
+#[path = "http_metrics_test.rs"]
+mod http_metrics_test;
+
+pub mod names {
+    /// Counter of HTTP requests by method + status class.
+    pub const REQUESTS_TOTAL: &str = "prover_http_requests_total";
+    /// Histogram of end-to-end HTTP request latency by method + status class.
+    pub const REQUEST_DURATION_SECONDS: &str = "prover_http_request_duration_seconds";
+    /// Gauge of in-flight HTTP requests.
+    pub const IN_FLIGHT_REQUESTS: &str = "prover_http_inflight_requests";
+}
+
+/// Pre-registers the HTTP metrics so the series exist before the first request.
+/// This describes the histogram without recording into it. A phantom 0-second
+/// observation would distort every quantile.
+pub fn preregister_http_metrics() {
+    // Pre-register the 2xx/4xx/5xx series so an error-rate alert reads zero instead of an
+    // absent series before the first failure. This calls the same label helpers as the live
+    // path, so the pre-registered label values match the emitted ones.
+    for status in [StatusCode::OK, StatusCode::BAD_REQUEST, StatusCode::INTERNAL_SERVER_ERROR] {
+        metrics::counter!(
+            names::REQUESTS_TOTAL,
+            "method" => method_label(&Method::POST),
+            "status" => status_label(status),
+        )
+        .increment(0);
+    }
+    metrics::describe_histogram!(
+        names::REQUEST_DURATION_SECONDS,
+        "HTTP request latency in seconds, by method and status class",
+    );
+    metrics::gauge!(names::IN_FLIGHT_REQUESTS).set(0.0);
+}
+
+/// tower [`Layer`] that records request count, latency, and the in-flight gauge
+/// for served requests.
+#[derive(Clone, Copy)]
+pub struct HttpMetricsLayer;
+
+impl<S> Layer<S> for HttpMetricsLayer {
+    type Service = HttpMetricsService<S>;
+
+    fn layer(&self, inner: S) -> Self::Service {
+        HttpMetricsService { inner }
+    }
+}
+
+/// tower [`Service`] produced by [`HttpMetricsLayer`].
+#[derive(Clone)]
+pub struct HttpMetricsService<S> {
+    inner: S,
+}
+
+impl<S, ReqB> Service<Request<ReqB>> for HttpMetricsService<S>
+where
+    S: Service<Request<ReqB>, Response = Response<HttpBody>>,
+    S::Future: Send + 'static,
+    S::Error: Send + 'static,
+{
+    type Response = Response<HttpBody>;
+    type Error = S::Error;
+    type Future = std::pin::Pin<
+        Box<dyn std::future::Future<Output = Result<Self::Response, Self::Error>> + Send>,
+    >;
+
+    fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
+        self.inner.poll_ready(cx)
+    }
+
+    fn call(&mut self, request: Request<ReqB>) -> Self::Future {
+        let method = method_label(request.method());
+        let start = Instant::now();
+        let future = self.inner.call(request);
+
+        Box::pin(async move {
+            let _in_flight_guard = GaugeGuard::acquire(names::IN_FLIGHT_REQUESTS);
+            let result = future.await;
+            let duration_seconds = start.elapsed().as_secs_f64();
+            let status = match &result {
+                Ok(response) => status_label(response.status()),
+                // The tower stack failed without producing an HTTP response.
+                Err(_) => "error",
+            };
+            metrics::histogram!(
+                names::REQUEST_DURATION_SECONDS,
+                "method" => method,
+                "status" => status,
+            )
+            .record(duration_seconds);
+            metrics::counter!(
+                names::REQUESTS_TOTAL,
+                "method" => method,
+                "status" => status,
+            )
+            .increment(1);
+            result
+        })
+    }
+}
+
+/// Collapses HTTP statuses into a bounded set of label values to cap Prometheus series cardinality.
+fn status_label(status: StatusCode) -> &'static str {
+    match status.as_u16() {
+        100..=199 => "1xx",
+        200..=299 => "2xx",
+        300..=399 => "3xx",
+        400..=499 => "4xx",
+        500..=599 => "5xx",
+        _ => "other",
+    }
+}
+
+/// Collapses HTTP methods into a bounded set of label values to cap Prometheus series cardinality.
+fn method_label(method: &Method) -> &'static str {
+    match *method {
+        Method::GET => "GET",
+        Method::POST => "POST",
+        Method::PUT => "PUT",
+        Method::DELETE => "DELETE",
+        Method::HEAD => "HEAD",
+        Method::OPTIONS => "OPTIONS",
+        Method::PATCH => "PATCH",
+        _ => "other",
+    }
+}

crates/starknet_transaction_prover/src/server/http_metrics_test.rs

@@ -0,0 +1,137 @@
+//! Unit tests for [`HttpMetricsLayer`].
+//!
+//! All tests share one process-global Prometheus recorder ([`shared_handle`]),
+//! so absolute sample values also reflect requests driven by other tests in this
+//! binary. Each assertion reads a baseline before the action and compares the
+//! delta.
+
+use bytes::Bytes;
+use http::{Method, Request, Response, StatusCode};
+use http_body_util::Full;
+use jsonrpsee::server::HttpBody;
+use tower::{Layer, ServiceBuilder, ServiceExt};
+use tower_http::compression::CompressionLayer;
+use tower_http::cors::CorsLayer;
+use tower_http::map_request_body::MapRequestBodyLayer;
+use tower_http::map_response_body::MapResponseBodyLayer;
+
+use crate::server::health::{HealthLayer, HEALTH_PATH};
+use crate::server::http_metrics::{names, HttpMetricsLayer};
+use crate::server::metrics::{MetricsLayer, METRICS_PATH};
+use crate::server::request_log::RequestLogLayer;
+use crate::server::request_span::RequestSpanLayer;
+use crate::server::test_recorder::{metric_value, shared_handle};
+use crate::server::OhttpJsonrpseeLayer;
+
+fn ok_service() -> impl tower::Service<
+    Request<HttpBody>,
+    Response = Response<HttpBody>,
+    Error = std::convert::Infallible,
+    Future = futures::future::Ready<Result<Response<HttpBody>, std::convert::Infallible>>,
+> + Clone {
+    tower::service_fn(|_req: Request<HttpBody>| {
+        let response = Response::builder()
+            .status(StatusCode::OK)
+            .body(HttpBody::new(Full::new(Bytes::new())))
+            .expect("static body is infallible");
+        futures::future::ready(Ok::<_, std::convert::Infallible>(response))
+    })
+}
+
+fn build_request(method: Method) -> Request<HttpBody> {
+    Request::builder()
+        .method(method)
+        .uri("/")
+        .body(HttpBody::new(Full::new(Bytes::new())))
+        .expect("static body is infallible")
+}
+
+#[tokio::test]
+async fn records_counter_histogram_and_returns_inflight_to_zero() {
+    let handle = shared_handle();
+    let svc = HttpMetricsLayer.layer(ok_service());
+
+    let scrape = handle.render();
+    let before_counter = metric_value(&scrape, &post_2xx_counter_line());
+    let before_histogram = metric_value(&scrape, &post_duration_count_line());
+
+    for _ in 0..3 {
+        let response = svc.clone().oneshot(build_request(Method::POST)).await.unwrap();
+        assert_eq!(response.status(), StatusCode::OK);
+    }
+
+    let scrape = handle.render();
+    assert_eq!(
+        metric_value(&scrape, &post_2xx_counter_line()) - before_counter,
+        3.0,
+        "counter delta"
+    );
+    assert!(
+        scrape.lines().any(|line| line.starts_with(&post_duration_count_line())),
+        "latency histogram must carry both method and status labels; scrape:\n{scrape}"
+    );
+    assert_eq!(
+        metric_value(&scrape, &post_duration_count_line()) - before_histogram,
+        3.0,
+        "histogram delta"
+    );
+
+    // The gauge is back at zero, so the guard ran for every request.
+    assert_eq!(metric_value(&scrape, names::IN_FLIGHT_REQUESTS), 0.0);
+}
+
+/// Sums the `prover_http_requests_total` series carrying `method="GET"`, across
+/// statuses. Scoped to GET because the recorder is process-global and other
+/// tests in this binary drive POSTs through the same layer in parallel. Only
+/// this test sends a GET through it.
+fn get_http_requests(scrape: &str) -> f64 {
+    scrape
+        .lines()
+        .filter(|line| !line.starts_with('#') && line.starts_with(names::REQUESTS_TOTAL))
+        .filter(|line| line.contains("method=\"GET\""))
+        .filter_map(|line| line.rsplit_once(' ').and_then(|(_, value)| value.parse::<f64>().ok()))
+        .sum()
+}
+
+fn post_2xx_counter_line() -> String {
+    format!("{}{{method=\"POST\",status=\"2xx\"}}", names::REQUESTS_TOTAL)
+}
+
+fn post_duration_count_line() -> String {
+    format!("{}_count{{method=\"POST\",status=\"2xx\"}}", names::REQUEST_DURATION_SECONDS)
+}
+
+/// `HttpMetricsLayer` sits below `HealthLayer` and `MetricsLayer` in the
+/// production chain so probe and scrape traffic stays out of the request
+/// distribution. Layer order is the only thing enforcing that, so this test
+/// fails if the order changes.
+#[tokio::test]
+async fn probe_and_scrape_traffic_is_excluded_from_http_metrics() {
+    let handle = shared_handle();
+    let svc = prover_http_middleware!(
+        MetricsLayer::new(handle.clone()),
+        None::<CorsLayer>,
+        None::<OhttpJsonrpseeLayer>,
+    )
+    .service(ok_service());
+
+    let before_get = get_http_requests(&handle.render());
+
+    for path in [HEALTH_PATH, METRICS_PATH] {
+        let request = Request::builder()
+            .method(Method::GET)
+            .uri(path)
+            .body(HttpBody::new(Full::new(Bytes::new())))
+            .expect("static body is infallible");
+        let response = svc.clone().oneshot(request).await.unwrap();
+        assert_eq!(response.status(), StatusCode::OK, "{path} should be served by its layer");
+    }
+
+    let after_get = get_http_requests(&handle.render());
+    assert_eq!(
+        after_get - before_get,
+        0.0,
+        "short-circuited {HEALTH_PATH}/{METRICS_PATH} traffic must not reach the HTTP metrics \
+         layer"
+    );
+}

crates/starknet_transaction_prover/src/server/metrics.rs

@@ -16,6 +16,8 @@ use jsonrpsee::server::HttpBody;
 use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle};
 use tower::{Layer, Service};
 
+use crate::server::http_metrics;
+
 #[cfg(test)]
 #[path = "metrics_test.rs"]
 mod metrics_test;
@@ -47,8 +49,19 @@ const DURATION_HISTOGRAM_BUCKETS: &[(&str, &[f64])] = &[
     (names::PROVE_TRANSACTION_DURATION_SECONDS, PROVING_DURATION_BUCKETS),
     (names::OS_RUN_DURATION_SECONDS, PROVING_DURATION_BUCKETS),
     (names::STWO_PROVE_DURATION_SECONDS, PROVING_DURATION_BUCKETS),
+    (http_metrics::names::REQUEST_DURATION_SECONDS, HTTP_DURATION_BUCKETS),
 ];
 
+/// Bucket bounds, in seconds, for the HTTP latency histogram. The layers above
+/// short-circuit probe and scrape traffic, so these buckets cover JSON-RPC
+/// calls. The range spans a millisecond-scale reject at one end and, at the
+/// other, a proving POST held open for its queue wait plus the proof itself.
+/// Boundaries at 2s and 10s match the proving histogram so a dashboard can read
+/// the two against each other, and the 30s boundary is the default queue-wait
+/// timeout.
+const HTTP_DURATION_BUCKETS: &[f64] =
+    &[0.005, 0.025, 0.1, 0.5, 1.0, 2.0, 5.0, 10.0, 20.0, 30.0, 45.0, 60.0];
+
 /// Metric name constants, so `metrics!` calls elsewhere point at one
 /// definition instead of repeating string literals.
 pub mod names {
@@ -99,6 +112,7 @@ pub fn install_exporter(version: &str, git_sha: &str) -> anyhow::Result<Promethe
         "git_sha" => git_sha.to_string(),
     )
     .set(1.0);
+    super::http_metrics::preregister_http_metrics();
     Ok(handle)
 }
 
@@ -114,6 +128,25 @@ pub fn spawn_upkeep(handle: PrometheusHandle) -> tokio::task::JoinHandle<()> {
     })
 }
 
+/// Increments a gauge on construction and decrements it on drop, so a panic or
+/// a dropped future cannot leak the gauge upward.
+pub struct GaugeGuard {
+    metric: &'static str,
+}
+
+impl GaugeGuard {
+    pub fn acquire(metric: &'static str) -> Self {
+        metrics::gauge!(metric).increment(1.0);
+        Self { metric }
+    }
+}
+
+impl Drop for GaugeGuard {
+    fn drop(&mut self) {
+        metrics::gauge!(self.metric).decrement(1.0);
+    }
+}
+
 #[derive(Clone)]
 pub struct MetricsLayer {
     handle: PrometheusHandle,

crates/starknet_transaction_prover/src/server/ohttp_integration_test.rs

@@ -32,7 +32,7 @@ use tower_ohttp::OhttpLayer;
 use crate::server::request_log::{RequestLogLayer, REQUEST_ID_HEADER};
 use crate::server::request_span::RequestSpanLayer;
 use crate::server::test_recorder::shared_handle;
-use crate::server::{HealthLayer, MetricsLayer, OHTTP_JSONRPSEE_BODY_BUILDER};
+use crate::server::{HealthLayer, HttpMetricsLayer, MetricsLayer, OHTTP_JSONRPSEE_BODY_BUILDER};
 
 const DEFAULT_BODY_LIMIT: usize = 102_400;
 const KEY_CACHE_SECS: u64 = 3600;