diff --git a/crates/starknet_transaction_prover/README.md b/crates/starknet_transaction_prover/README.md index b50f263d8eb..2450cb48866 100644 --- a/crates/starknet_transaction_prover/README.md +++ b/crates/starknet_transaction_prover/README.md @@ -300,6 +300,9 @@ scrapes bypass CORS and JSON-RPC parsing, and the endpoint is unauthenticated. | Metric | Type | Labels | Description | |---|---|---|---| | `prover_build_info` | gauge | `version`, `git_sha` | Always 1. Identifies the running build from a scrape. `git_sha` comes from the `GIT_SHA` docker build arg, and is `unknown` when the build doesn't pass one. | +| `prover_http_requests_total` | counter | `method`, `status` | HTTP request count. `method` is a bounded enum (`GET`/`POST`/`PUT`/`DELETE`/`HEAD`/`OPTIONS`/`PATCH`/`other`); `status` is the HTTP status class (`1xx` through `5xx`, plus `other`, and `error` when the tower stack produced no response). Excludes `/health` and `/metrics` probes. | +| `prover_http_request_duration_seconds` | histogram | `method`, `status` | End-to-end HTTP request latency, using the same bounded `method` and `status` label values as the request counter. Excludes `/health` and `/metrics` probes. | +| `prover_http_inflight_requests` | gauge | none | Current count of HTTP requests being handled. Decremented via RAII so panics and cancellations don't leak. | | `prover_prove_transaction_outcome_total` | counter | `outcome` | Every proving request that reaches the prover, so its total is the shared denominator for all proving rates. `outcome` is a bounded enum: `success`, `failure_validation`, `failure_blocked`, `failure_runner`, `failure_output_parse`, `failure_proving`. | | `prover_prove_transaction_duration_seconds` | histogram | `outcome` | Duration of the whole proving call, covering input validation, the optional blocking check, the virtual OS run and proving. Recorded for failures too, so filter on `outcome` for success-only percentiles. | | `prover_os_run_duration_seconds` | histogram | none | Virtual OS execution time, recorded for successful runs only. | diff --git a/crates/starknet_transaction_prover/src/server.rs b/crates/starknet_transaction_prover/src/server.rs index 5b51468b050..5f97ec12327 100644 --- a/crates/starknet_transaction_prover/src/server.rs +++ b/crates/starknet_transaction_prover/src/server.rs @@ -43,6 +43,8 @@ pub const OHTTP_JSONRPSEE_BODY_BUILDER: fn(Full) -> HttpBody = HttpBody:: /// - `RequestLogLayer` is outermost so the latency it measures covers every other layer. /// - `HealthLayer` and `MetricsLayer` sit inside it so probes and scrapes complete before /// CORS/OHTTP. +/// - `HttpMetricsLayer` records per-request latency. It sits below `HealthLayer` and +/// `MetricsLayer`, so the probes and scrapes they short-circuit stay out of the distribution. /// - `OhttpLayer` must sit OUTSIDE `CompressionLayer` so compression applies to the inner JSON-RPC /// response (the client's inner `Accept-Encoding` travels through BHTTP into jsonrpsee) rather /// than to the OHTTP ciphertext envelope. `MapRequestBodyLayer`/`MapResponseBodyLayer` keep @@ -56,6 +58,7 @@ macro_rules! prover_http_middleware { .layer(RequestLogLayer) .layer(HealthLayer) .layer($metrics_layer) + .layer(HttpMetricsLayer) .option_layer($cors_layer) .layer(MapRequestBodyLayer::new(HttpBody::new)) .option_layer($ohttp_layer) @@ -69,6 +72,7 @@ pub mod config; pub mod cors; pub mod errors; pub mod health; +pub mod http_metrics; pub mod log_redact; pub mod metrics; #[cfg(test)] @@ -86,6 +90,7 @@ pub mod test_recorder; pub mod tls; pub use health::{HealthLayer, HEALTH_PATH}; +pub use http_metrics::HttpMetricsLayer; pub use metrics::{MetricsLayer, METRICS_PATH}; pub use request_log::{RequestLogLayer, REQUEST_ID_HEADER}; pub use request_span::RequestSpanLayer; diff --git a/crates/starknet_transaction_prover/src/server/http_metrics.rs b/crates/starknet_transaction_prover/src/server/http_metrics.rs new file mode 100644 index 00000000000..ef4701fcc69 --- /dev/null +++ b/crates/starknet_transaction_prover/src/server/http_metrics.rs @@ -0,0 +1,138 @@ +//! tower middleware that records HTTP-level Prometheus metrics: +//! request count, latency histogram, and an RAII-guarded in-flight gauge. + +use std::task::{Context, Poll}; +use std::time::Instant; + +use http::{Method, Request, Response, StatusCode}; +use jsonrpsee::server::HttpBody; +use tower::{Layer, Service}; + +use crate::server::metrics::GaugeGuard; + +#[cfg(test)] +#[path = "http_metrics_test.rs"] +mod http_metrics_test; + +pub mod names { + /// Counter of HTTP requests by method + status class. + pub const REQUESTS_TOTAL: &str = "prover_http_requests_total"; + /// Histogram of end-to-end HTTP request latency by method + status class. + pub const REQUEST_DURATION_SECONDS: &str = "prover_http_request_duration_seconds"; + /// Gauge of in-flight HTTP requests. + pub const IN_FLIGHT_REQUESTS: &str = "prover_http_inflight_requests"; +} + +/// Pre-registers the HTTP metrics so the series exist before the first request. +/// This describes the histogram without recording into it. A phantom 0-second +/// observation would distort every quantile. +pub fn preregister_http_metrics() { + // Pre-register the 2xx/4xx/5xx series so an error-rate alert reads zero instead of an + // absent series before the first failure. This calls the same label helpers as the live + // path, so the pre-registered label values match the emitted ones. + for status in [StatusCode::OK, StatusCode::BAD_REQUEST, StatusCode::INTERNAL_SERVER_ERROR] { + metrics::counter!( + names::REQUESTS_TOTAL, + "method" => method_label(&Method::POST), + "status" => status_label(status), + ) + .increment(0); + } + metrics::describe_histogram!( + names::REQUEST_DURATION_SECONDS, + "HTTP request latency in seconds, by method and status class", + ); + metrics::gauge!(names::IN_FLIGHT_REQUESTS).set(0.0); +} + +/// tower [`Layer`] that records request count, latency, and the in-flight gauge +/// for served requests. +#[derive(Clone, Copy)] +pub struct HttpMetricsLayer; + +impl Layer for HttpMetricsLayer { + type Service = HttpMetricsService; + + fn layer(&self, inner: S) -> Self::Service { + HttpMetricsService { inner } + } +} + +/// tower [`Service`] produced by [`HttpMetricsLayer`]. +#[derive(Clone)] +pub struct HttpMetricsService { + inner: S, +} + +impl Service> for HttpMetricsService +where + S: Service, Response = Response>, + S::Future: Send + 'static, + S::Error: Send + 'static, +{ + type Response = Response; + type Error = S::Error; + type Future = std::pin::Pin< + Box> + Send>, + >; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx) + } + + fn call(&mut self, request: Request) -> Self::Future { + let method = method_label(request.method()); + let start = Instant::now(); + let future = self.inner.call(request); + + Box::pin(async move { + let _in_flight_guard = GaugeGuard::acquire(names::IN_FLIGHT_REQUESTS); + let result = future.await; + let duration_seconds = start.elapsed().as_secs_f64(); + let status = match &result { + Ok(response) => status_label(response.status()), + // The tower stack failed without producing an HTTP response. + Err(_) => "error", + }; + metrics::histogram!( + names::REQUEST_DURATION_SECONDS, + "method" => method, + "status" => status, + ) + .record(duration_seconds); + metrics::counter!( + names::REQUESTS_TOTAL, + "method" => method, + "status" => status, + ) + .increment(1); + result + }) + } +} + +/// Collapses HTTP statuses into a bounded set of label values to cap Prometheus series cardinality. +fn status_label(status: StatusCode) -> &'static str { + match status.as_u16() { + 100..=199 => "1xx", + 200..=299 => "2xx", + 300..=399 => "3xx", + 400..=499 => "4xx", + 500..=599 => "5xx", + _ => "other", + } +} + +/// Collapses HTTP methods into a bounded set of label values to cap Prometheus series cardinality. +fn method_label(method: &Method) -> &'static str { + match *method { + Method::GET => "GET", + Method::POST => "POST", + Method::PUT => "PUT", + Method::DELETE => "DELETE", + Method::HEAD => "HEAD", + Method::OPTIONS => "OPTIONS", + Method::PATCH => "PATCH", + _ => "other", + } +} diff --git a/crates/starknet_transaction_prover/src/server/http_metrics_test.rs b/crates/starknet_transaction_prover/src/server/http_metrics_test.rs new file mode 100644 index 00000000000..e8a6d298606 --- /dev/null +++ b/crates/starknet_transaction_prover/src/server/http_metrics_test.rs @@ -0,0 +1,137 @@ +//! Unit tests for [`HttpMetricsLayer`]. +//! +//! All tests share one process-global Prometheus recorder ([`shared_handle`]), +//! so absolute sample values also reflect requests driven by other tests in this +//! binary. Each assertion reads a baseline before the action and compares the +//! delta. + +use bytes::Bytes; +use http::{Method, Request, Response, StatusCode}; +use http_body_util::Full; +use jsonrpsee::server::HttpBody; +use tower::{Layer, ServiceBuilder, ServiceExt}; +use tower_http::compression::CompressionLayer; +use tower_http::cors::CorsLayer; +use tower_http::map_request_body::MapRequestBodyLayer; +use tower_http::map_response_body::MapResponseBodyLayer; + +use crate::server::health::{HealthLayer, HEALTH_PATH}; +use crate::server::http_metrics::{names, HttpMetricsLayer}; +use crate::server::metrics::{MetricsLayer, METRICS_PATH}; +use crate::server::request_log::RequestLogLayer; +use crate::server::request_span::RequestSpanLayer; +use crate::server::test_recorder::{metric_value, shared_handle}; +use crate::server::OhttpJsonrpseeLayer; + +fn ok_service() -> impl tower::Service< + Request, + Response = Response, + Error = std::convert::Infallible, + Future = futures::future::Ready, std::convert::Infallible>>, +> + Clone { + tower::service_fn(|_req: Request| { + let response = Response::builder() + .status(StatusCode::OK) + .body(HttpBody::new(Full::new(Bytes::new()))) + .expect("static body is infallible"); + futures::future::ready(Ok::<_, std::convert::Infallible>(response)) + }) +} + +fn build_request(method: Method) -> Request { + Request::builder() + .method(method) + .uri("/") + .body(HttpBody::new(Full::new(Bytes::new()))) + .expect("static body is infallible") +} + +#[tokio::test] +async fn records_counter_histogram_and_returns_inflight_to_zero() { + let handle = shared_handle(); + let svc = HttpMetricsLayer.layer(ok_service()); + + let scrape = handle.render(); + let before_counter = metric_value(&scrape, &post_2xx_counter_line()); + let before_histogram = metric_value(&scrape, &post_duration_count_line()); + + for _ in 0..3 { + let response = svc.clone().oneshot(build_request(Method::POST)).await.unwrap(); + assert_eq!(response.status(), StatusCode::OK); + } + + let scrape = handle.render(); + assert_eq!( + metric_value(&scrape, &post_2xx_counter_line()) - before_counter, + 3.0, + "counter delta" + ); + assert!( + scrape.lines().any(|line| line.starts_with(&post_duration_count_line())), + "latency histogram must carry both method and status labels; scrape:\n{scrape}" + ); + assert_eq!( + metric_value(&scrape, &post_duration_count_line()) - before_histogram, + 3.0, + "histogram delta" + ); + + // The gauge is back at zero, so the guard ran for every request. + assert_eq!(metric_value(&scrape, names::IN_FLIGHT_REQUESTS), 0.0); +} + +/// Sums the `prover_http_requests_total` series carrying `method="GET"`, across +/// statuses. Scoped to GET because the recorder is process-global and other +/// tests in this binary drive POSTs through the same layer in parallel. Only +/// this test sends a GET through it. +fn get_http_requests(scrape: &str) -> f64 { + scrape + .lines() + .filter(|line| !line.starts_with('#') && line.starts_with(names::REQUESTS_TOTAL)) + .filter(|line| line.contains("method=\"GET\"")) + .filter_map(|line| line.rsplit_once(' ').and_then(|(_, value)| value.parse::().ok())) + .sum() +} + +fn post_2xx_counter_line() -> String { + format!("{}{{method=\"POST\",status=\"2xx\"}}", names::REQUESTS_TOTAL) +} + +fn post_duration_count_line() -> String { + format!("{}_count{{method=\"POST\",status=\"2xx\"}}", names::REQUEST_DURATION_SECONDS) +} + +/// `HttpMetricsLayer` sits below `HealthLayer` and `MetricsLayer` in the +/// production chain so probe and scrape traffic stays out of the request +/// distribution. Layer order is the only thing enforcing that, so this test +/// fails if the order changes. +#[tokio::test] +async fn probe_and_scrape_traffic_is_excluded_from_http_metrics() { + let handle = shared_handle(); + let svc = prover_http_middleware!( + MetricsLayer::new(handle.clone()), + None::, + None::, + ) + .service(ok_service()); + + let before_get = get_http_requests(&handle.render()); + + for path in [HEALTH_PATH, METRICS_PATH] { + let request = Request::builder() + .method(Method::GET) + .uri(path) + .body(HttpBody::new(Full::new(Bytes::new()))) + .expect("static body is infallible"); + let response = svc.clone().oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::OK, "{path} should be served by its layer"); + } + + let after_get = get_http_requests(&handle.render()); + assert_eq!( + after_get - before_get, + 0.0, + "short-circuited {HEALTH_PATH}/{METRICS_PATH} traffic must not reach the HTTP metrics \ + layer" + ); +} diff --git a/crates/starknet_transaction_prover/src/server/metrics.rs b/crates/starknet_transaction_prover/src/server/metrics.rs index 8b4a9488bc8..0fc94d417d4 100644 --- a/crates/starknet_transaction_prover/src/server/metrics.rs +++ b/crates/starknet_transaction_prover/src/server/metrics.rs @@ -16,6 +16,8 @@ use jsonrpsee::server::HttpBody; use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle}; use tower::{Layer, Service}; +use crate::server::http_metrics; + #[cfg(test)] #[path = "metrics_test.rs"] mod metrics_test; @@ -47,8 +49,19 @@ const DURATION_HISTOGRAM_BUCKETS: &[(&str, &[f64])] = &[ (names::PROVE_TRANSACTION_DURATION_SECONDS, PROVING_DURATION_BUCKETS), (names::OS_RUN_DURATION_SECONDS, PROVING_DURATION_BUCKETS), (names::STWO_PROVE_DURATION_SECONDS, PROVING_DURATION_BUCKETS), + (http_metrics::names::REQUEST_DURATION_SECONDS, HTTP_DURATION_BUCKETS), ]; +/// Bucket bounds, in seconds, for the HTTP latency histogram. The layers above +/// short-circuit probe and scrape traffic, so these buckets cover JSON-RPC +/// calls. The range spans a millisecond-scale reject at one end and, at the +/// other, a proving POST held open for its queue wait plus the proof itself. +/// Boundaries at 2s and 10s match the proving histogram so a dashboard can read +/// the two against each other, and the 30s boundary is the default queue-wait +/// timeout. +const HTTP_DURATION_BUCKETS: &[f64] = + &[0.005, 0.025, 0.1, 0.5, 1.0, 2.0, 5.0, 10.0, 20.0, 30.0, 45.0, 60.0]; + /// Metric name constants, so `metrics!` calls elsewhere point at one /// definition instead of repeating string literals. pub mod names { @@ -99,6 +112,7 @@ pub fn install_exporter(version: &str, git_sha: &str) -> anyhow::Result git_sha.to_string(), ) .set(1.0); + super::http_metrics::preregister_http_metrics(); Ok(handle) } @@ -114,6 +128,25 @@ pub fn spawn_upkeep(handle: PrometheusHandle) -> tokio::task::JoinHandle<()> { }) } +/// Increments a gauge on construction and decrements it on drop, so a panic or +/// a dropped future cannot leak the gauge upward. +pub struct GaugeGuard { + metric: &'static str, +} + +impl GaugeGuard { + pub fn acquire(metric: &'static str) -> Self { + metrics::gauge!(metric).increment(1.0); + Self { metric } + } +} + +impl Drop for GaugeGuard { + fn drop(&mut self) { + metrics::gauge!(self.metric).decrement(1.0); + } +} + #[derive(Clone)] pub struct MetricsLayer { handle: PrometheusHandle, diff --git a/crates/starknet_transaction_prover/src/server/ohttp_integration_test.rs b/crates/starknet_transaction_prover/src/server/ohttp_integration_test.rs index 38f9b5a8c18..78cd8c0f138 100644 --- a/crates/starknet_transaction_prover/src/server/ohttp_integration_test.rs +++ b/crates/starknet_transaction_prover/src/server/ohttp_integration_test.rs @@ -32,7 +32,7 @@ use tower_ohttp::OhttpLayer; use crate::server::request_log::{RequestLogLayer, REQUEST_ID_HEADER}; use crate::server::request_span::RequestSpanLayer; use crate::server::test_recorder::shared_handle; -use crate::server::{HealthLayer, MetricsLayer, OHTTP_JSONRPSEE_BODY_BUILDER}; +use crate::server::{HealthLayer, HttpMetricsLayer, MetricsLayer, OHTTP_JSONRPSEE_BODY_BUILDER}; const DEFAULT_BODY_LIMIT: usize = 102_400; const KEY_CACHE_SECS: u64 = 3600; diff --git a/crates/starknet_transaction_prover/src/server/tls.rs b/crates/starknet_transaction_prover/src/server/tls.rs index da06f29c456..74fa6e58d00 100644 --- a/crates/starknet_transaction_prover/src/server/tls.rs +++ b/crates/starknet_transaction_prover/src/server/tls.rs @@ -29,7 +29,13 @@ use tower_http::map_request_body::MapRequestBodyLayer; use tower_http::map_response_body::MapResponseBodyLayer; use tracing::warn; -use crate::server::{HealthLayer, RequestLogLayer, RequestSpanLayer, ServerLayers}; +use crate::server::{ + HealthLayer, + HttpMetricsLayer, + RequestLogLayer, + RequestSpanLayer, + ServerLayers, +}; #[cfg(test)] #[path = "tls_test.rs"]