Skip to content

Commit 1c0c806

Browse files
tigrantsat-openaicopyberry
authored andcommitted
Record MCP tool call outcomes in traces (#62833)
## Why A successful JSON-RPC exchange can still return a failed tool result. MCP error metrics count that failure, but the trace span can lack an error status. Dropped calls can also leave spans without an outcome. ## What changed Record `codex.mcp.call.outcome` on `mcp.tools.call` spans and mark failures with OpenTelemetry error status using the same classification as the metrics. Record `interrupted` when the call future is dropped before completion, including before its first poll. Capture the numeric JSON-RPC error code in `rpc.jsonrpc.error_code` when available, leaving server error messages and data out of the span. ## Testing Extend in-memory telemetry tests to check that spans and metrics agree on error classification and that span attributes omit sensitive test values. Cover success and cancellation before the first poll and while the call is pending. GitOrigin-RevId: 57fcdc36f19cfc2f227c1ec964a932f3cf7eaa04
1 parent a1afa12 commit 1c0c806

3 files changed

Lines changed: 234 additions & 38 deletions

File tree

‎codex-rs/core/src/mcp_tool_call.rs‎

Lines changed: 55 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -508,6 +508,7 @@ async fn handle_approved_mcp_tool_call(
508508
.unwrap_or_else(|| JsonValue::Object(serde_json::Map::new()));
509509
let mut elicitation_type = None;
510510
let mut request_error_code = None;
511+
let mut request_rpc_error_code = None;
511512
let mut admission_error = None;
512513
let admit = || {
513514
let reason = sess
@@ -525,6 +526,19 @@ async fn handle_approved_mcp_tool_call(
525526
&prepared_call,
526527
turn_context.config.mcp_servers.get().get(&server),
527528
);
529+
let span = mcp_tool_call_span(
530+
sess,
531+
turn_context,
532+
McpToolCallSpanFields {
533+
server_name: &server,
534+
tool_name: &tool_name,
535+
call_id,
536+
server_origin: server_origin.as_deref(),
537+
connector_id,
538+
connector_name,
539+
},
540+
);
541+
let outcome_guard = McpCallSpanOutcomeGuard::new(span.clone());
528542
let result = async {
529543
let result = async {
530544
let result = prepared_call
@@ -667,6 +681,10 @@ async fn handle_approved_mcp_tool_call(
667681
}
668682
Err(error) => {
669683
request_error_code = codex_rmcp_client::request_error_code(&error);
684+
// Preserve only the numeric protocol code, never the server's
685+
// message or arbitrary data (which may contain user content).
686+
request_rpc_error_code =
687+
codex_rmcp_client::mcp_error(&error).map(|error| error.code.0);
670688
format!("tool call error: {error:?}")
671689
}
672690
});
@@ -714,21 +732,13 @@ async fn handle_approved_mcp_tool_call(
714732
.await)
715733
}
716734
.await;
717-
record_mcp_result_span_telemetry(&Span::current(), &result, request_error_code);
735+
outcome_guard.complete(&result, request_error_code);
736+
if let Some(code) = request_rpc_error_code {
737+
Span::current().record("rpc.jsonrpc.error_code", i64::from(code));
738+
}
718739
result
719740
}
720-
.instrument(mcp_tool_call_span(
721-
sess,
722-
turn_context,
723-
McpToolCallSpanFields {
724-
server_name: &server,
725-
tool_name: &tool_name,
726-
call_id,
727-
server_origin: server_origin.as_deref(),
728-
connector_id,
729-
connector_name,
730-
},
731-
))
741+
.instrument(span)
732742
.await;
733743
if let Some(error) = admission_error {
734744
notify_mcp_tool_call_completed(
@@ -811,6 +821,9 @@ fn mcp_tool_call_span(
811821
codex.mcp.server_user_flow.triggered = Empty,
812822
error.type = Empty,
813823
codex.mcp.error.code = Empty,
824+
codex.mcp.call.outcome = Empty,
825+
rpc.jsonrpc.error_code = Empty,
826+
otel.status_code = Empty,
814827
);
815828
record_server_fields(&span, fields.server_origin);
816829
span
@@ -840,6 +853,35 @@ fn record_server_fields(span: &Span, url: Option<&str>) {
840853
}
841854
}
842855

856+
/// Records one outcome, including when the instrumented call is dropped before its first poll.
857+
struct McpCallSpanOutcomeGuard {
858+
span: Option<Span>,
859+
}
860+
861+
impl McpCallSpanOutcomeGuard {
862+
fn new(span: Span) -> Self {
863+
Self { span: Some(span) }
864+
}
865+
866+
fn complete(
867+
mut self,
868+
result: &Result<CallToolResult, String>,
869+
request_error_code: Option<&str>,
870+
) {
871+
if let Some(span) = self.span.take() {
872+
record_mcp_result_span_telemetry(&span, result, request_error_code);
873+
}
874+
}
875+
}
876+
877+
impl Drop for McpCallSpanOutcomeGuard {
878+
fn drop(&mut self) {
879+
if let Some(span) = &self.span {
880+
span.record("codex.mcp.call.outcome", "interrupted");
881+
}
882+
}
883+
}
884+
843885
fn record_mcp_result_span_telemetry(
844886
span: &Span,
845887
result: &Result<CallToolResult, String>,

‎codex-rs/core/src/mcp_tool_call/telemetry.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,12 @@ fn allowlisted_codex_telemetry_error_code(code: &str) -> Option<&str> {
214214
}
215215

216216
pub(super) fn record_mcp_call_outcome_span_telemetry(span: &Span, outcome: &McpCallMetricOutcome) {
217+
span.record("codex.mcp.call.outcome", outcome.status);
218+
// A successful JSON-RPC exchange can still contain a failed tool result. Keep the
219+
// exported span status consistent with the outcome used by MCP error metrics.
220+
if outcome.status == "error" {
221+
span.record("otel.status_code", "ERROR");
222+
}
217223
let (Some(error_type), Some(error_code)) = (outcome.error_type, outcome.error_code.as_deref())
218224
else {
219225
return;

‎codex-rs/core/src/mcp_tool_call_tests.rs‎

Lines changed: 173 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ use codex_features::Features;
3030
use codex_hooks::HooksConfig;
3131
use codex_mcp::McpServerKey;
3232
use codex_model_provider::create_model_provider;
33+
use codex_otel::MetricsClient;
34+
use codex_otel::MetricsConfig;
3335
use codex_protocol::ResponseItemId;
3436
use codex_protocol::models::PermissionProfile;
3537
use codex_protocol::openai_models::ReasoningEffort as ReasoningEffortConfig;
@@ -54,10 +56,13 @@ use core_test_support::responses::sse;
5456
use core_test_support::responses::start_mock_server;
5557
use ctor::ctor;
5658
use opentelemetry::trace::TracerProvider;
59+
use opentelemetry_sdk::metrics::data::AggregatedMetrics;
60+
use opentelemetry_sdk::metrics::data::MetricData;
5761
use opentelemetry_sdk::trace::InMemorySpanExporter;
5862
use opentelemetry_sdk::trace::SdkTracerProvider;
5963
use pretty_assertions::assert_eq;
6064
use serde::Deserialize;
65+
use std::collections::BTreeMap;
6166
use std::collections::HashMap;
6267
use std::io::Write;
6368
use std::sync::Arc;
@@ -566,10 +571,12 @@ async fn mcp_tool_call_span_records_expected_fields() {
566571
}
567572

568573
#[test_case::test_case(Some("RATE_LIMITED"), None, None, "RATE_LIMITED"; "structured error")]
574+
#[test_case::test_case(Some("SITE_AUTH_REQUIRED"), None, None, "SITE_AUTH_REQUIRED"; "server defined site auth error is retained")]
575+
#[test_case::test_case(None, Some("execution_rejected"), None, "execution_rejected"; "runtime rejection is recorded as a failure")]
569576
#[test_case::test_case(None, Some("credential-sentinel"), None, "unknown"; "sensitive metadata")]
570577
#[test_case::test_case(None, None, Some("transport_closed"), "transport_closed"; "closed transport")]
571578
#[tokio::test]
572-
async fn mcp_tool_call_span_records_error_type_and_error_code(
579+
async fn mcp_tool_call_exports_intended_error_to_metrics_and_span(
573580
structured_code: Option<&str>,
574581
metadata_code: Option<&str>,
575582
request_error_code: Option<&str>,
@@ -584,9 +591,20 @@ async fn mcp_tool_call_span_records_error_type_and_error_code(
584591
let _guard = tracing::subscriber::set_default(subscriber);
585592

586593
let (session, turn_context) = make_session_and_context().await;
594+
let metrics = MetricsClient::new(
595+
MetricsConfig::in_memory("test", "mcp-error-test", "1", Default::default())
596+
.with_runtime_reader(),
597+
)
598+
.expect("metrics client");
599+
let telemetry = turn_context
600+
.session_telemetry
601+
.clone()
602+
.with_metrics_without_metadata_tags(metrics.clone());
587603
let result = Ok(CallToolResult {
588-
content: Vec::new(),
589-
structured_content: structured_code.map(|code| serde_json::json!({"error_code": code})),
604+
content: vec![serde_json::json!({"type": "text", "text": "private-message-sentinel"})],
605+
structured_content: structured_code.map(
606+
|code| serde_json::json!({"error_code": code, "message": "private-message-sentinel"}),
607+
),
590608
is_error: Some(true),
591609
meta: metadata_code.map(|code| {
592610
serde_json::json!({
@@ -603,27 +621,83 @@ async fn mcp_tool_call_span_records_error_type_and_error_code(
603621
&session,
604622
&turn_context,
605623
McpToolCallSpanFields {
606-
server_name: CODEX_APPS_MCP_SERVER_NAME,
607-
tool_name: "calendar_search",
624+
server_name: "sites-test-server",
625+
tool_name: "visit_site",
608626
call_id: "call-123",
609-
server_origin: Some("https://chatgpt.com/api/codex/ps/mcp"),
610-
connector_id: Some("calendar"),
611-
connector_name: Some("Calendar"),
627+
server_origin: None,
628+
connector_id: None,
629+
connector_name: None,
612630
},
613631
);
614632

615-
async {
616-
record_mcp_result_span_telemetry(&Span::current(), &result, request_error_code);
633+
let outcome_guard = McpCallSpanOutcomeGuard::new(span.clone());
634+
async { outcome_guard.complete(&result, request_error_code) }
635+
.instrument(span)
636+
.await;
637+
638+
let outcome = mcp_call_metric_outcome(&result).with_request_error_code(request_error_code);
639+
emit_mcp_call_metrics(
640+
&telemetry,
641+
&outcome,
642+
"sites-test-server",
643+
"visit_site",
644+
/*connector_id*/ None,
645+
/*connector_name*/ None,
646+
/*duration*/ None,
647+
);
648+
let error_type = if request_error_code.is_some() {
649+
"mcp_request"
650+
} else {
651+
"tool_result"
652+
};
653+
let snapshot = metrics.snapshot().expect("metric snapshot");
654+
let mut error_counts = Vec::new();
655+
for metric in snapshot
656+
.scope_metrics()
657+
.flat_map(opentelemetry_sdk::metrics::data::ScopeMetrics::metrics)
658+
.filter(|metric| metric.name() == "codex.mcp.call.error")
659+
{
660+
let AggregatedMetrics::U64(MetricData::Sum(sum)) = metric.data() else {
661+
panic!("MCP errors must be a counter");
662+
};
663+
for point in sum.data_points() {
664+
error_counts.push((
665+
point.value(),
666+
point
667+
.attributes()
668+
.map(|attribute| (attribute.key.to_string(), attribute.value.to_string()))
669+
.collect::<BTreeMap<_, _>>(),
670+
));
671+
}
617672
}
618-
.instrument(span)
619-
.await;
673+
assert_eq!(
674+
error_counts,
675+
vec![(
676+
1,
677+
BTreeMap::from([
678+
("status".into(), "error".into()),
679+
("server".into(), "sites-test-server".into()),
680+
("tool".into(), "visit_site".into()),
681+
("error_type".into(), error_type.into()),
682+
("error_code".into(), expected_code.into()),
683+
]),
684+
)],
685+
);
620686

621687
provider.force_flush().expect("flush spans");
622688
let spans = exporter.get_finished_spans().expect("exported spans");
623689
let span = spans
624690
.iter()
625691
.find(|span| span.name == "mcp.tools.call")
626692
.expect("MCP tool span");
693+
assert_eq!(span.status, opentelemetry::trace::Status::error(""));
694+
let outcomes = span
695+
.attributes
696+
.iter()
697+
.filter(|attribute| attribute.key.as_str() == "codex.mcp.call.outcome")
698+
.map(|attribute| attribute.value.to_string())
699+
.collect::<Vec<_>>();
700+
assert_eq!(outcomes, vec!["error".to_string()]);
627701
let error_codes = span
628702
.attributes
629703
.iter()
@@ -637,19 +711,96 @@ async fn mcp_tool_call_span_records_error_type_and_error_code(
637711
.filter(|attribute| attribute.key.as_str() == "error.type")
638712
.map(|attribute| attribute.value.to_string())
639713
.collect::<Vec<_>>();
640-
assert_eq!(
641-
error_types,
642-
vec![if request_error_code.is_some() {
643-
"mcp_request".to_string()
644-
} else {
645-
"tool_result".to_string()
646-
}]
647-
);
714+
assert_eq!(error_types, vec![error_type.to_string()]);
648715
assert!(
649716
span.attributes
650717
.iter()
651718
.all(|attribute| !attribute.value.to_string().contains("credential-sentinel"))
652719
);
720+
assert!(span.attributes.iter().all(|attribute| {
721+
!attribute
722+
.value
723+
.to_string()
724+
.contains("private-message-sentinel")
725+
}));
726+
}
727+
728+
enum McpCallSpanCompletion {
729+
Success,
730+
CancelBeforePoll,
731+
CancelWhilePending,
732+
}
733+
734+
#[test_case::test_case(McpCallSpanCompletion::Success, "ok"; "successful call")]
735+
#[test_case::test_case(McpCallSpanCompletion::CancelBeforePoll, "interrupted"; "cancelled before first poll")]
736+
#[test_case::test_case(McpCallSpanCompletion::CancelWhilePending, "interrupted"; "cancelled while pending")]
737+
#[tokio::test]
738+
async fn mcp_tool_call_span_exports_one_completion_or_cancellation_outcome(
739+
completion: McpCallSpanCompletion,
740+
expected_outcome: &str,
741+
) {
742+
let exporter = InMemorySpanExporter::default();
743+
let provider = SdkTracerProvider::builder()
744+
.with_simple_exporter(exporter.clone())
745+
.build();
746+
let subscriber = tracing_subscriber::registry()
747+
.with(tracing_opentelemetry::layer().with_tracer(provider.tracer("mcp-outcome-test")));
748+
let _guard = tracing::subscriber::set_default(subscriber);
749+
let (session, turn_context) = make_session_and_context().await;
750+
let span = mcp_tool_call_span(
751+
&session,
752+
&turn_context,
753+
McpToolCallSpanFields {
754+
server_name: "rmcp",
755+
tool_name: "echo",
756+
call_id: "call-123",
757+
server_origin: None,
758+
connector_id: None,
759+
connector_name: None,
760+
},
761+
);
762+
let outcome_guard = McpCallSpanOutcomeGuard::new(span.clone());
763+
let (complete_tx, complete_rx) = tokio::sync::oneshot::channel();
764+
let call = async {
765+
complete_rx.await.expect("release call");
766+
outcome_guard.complete(
767+
&Ok(CallToolResult {
768+
content: Vec::new(),
769+
structured_content: None,
770+
is_error: None,
771+
meta: None,
772+
}),
773+
/*request_error_code*/ None,
774+
);
775+
}
776+
.instrument(span);
777+
match completion {
778+
McpCallSpanCompletion::Success => {
779+
complete_tx.send(()).expect("complete call");
780+
call.await;
781+
}
782+
McpCallSpanCompletion::CancelBeforePoll => drop(call),
783+
McpCallSpanCompletion::CancelWhilePending => {
784+
let mut call = Box::pin(call);
785+
assert!(futures::poll!(call.as_mut()).is_pending());
786+
drop(call);
787+
}
788+
}
789+
790+
provider.force_flush().expect("flush spans");
791+
let spans = exporter.get_finished_spans().expect("exported spans");
792+
let span = spans
793+
.iter()
794+
.find(|span| span.name == "mcp.tools.call")
795+
.expect("MCP tool span");
796+
let outcomes = span
797+
.attributes
798+
.iter()
799+
.filter(|attribute| attribute.key.as_str() == "codex.mcp.call.outcome")
800+
.map(|attribute| attribute.value.to_string())
801+
.collect::<Vec<_>>();
802+
assert_eq!(outcomes, vec![expected_outcome.to_string()]);
803+
assert_eq!(span.status, opentelemetry::trace::Status::Unset);
653804
}
654805

655806
async fn mcp_result_telemetry_span_logs(meta: Option<serde_json::Value>) -> String {
@@ -686,12 +837,9 @@ async fn mcp_result_telemetry_span_logs(meta: Option<serde_json::Value>) -> Stri
686837
},
687838
);
688839

840+
let outcome_guard = McpCallSpanOutcomeGuard::new(span.clone());
689841
async {
690-
record_mcp_result_span_telemetry(
691-
&Span::current(),
692-
&result,
693-
/*request_error_code*/ None,
694-
);
842+
outcome_guard.complete(&result, /*request_error_code*/ None)
695843
}
696844
.instrument(span)
697845
.await;

0 commit comments

Comments
 (0)