Add auth 401 observability to client bug reports (#14611)

CXC-392

  [With
  401](https://openai.sentry.io/issues/7333870443/?project=4510195390611458&query=019ce8f8-560c-7f10-a00a-c59553740674&referrer=issue-stream)
  <img width="1909" height="555" alt="401 auth tags in Sentry"
  src="https://github.com/user-attachments/assets/412ea950-61c4-4780-9697-15c270971ee3"
  />


  - auth_401_*: preserved facts from the latest unauthorized response snapshot
  - auth_*: latest auth-related facts from the latest request attempt
  - auth_recovery_*: unauthorized recovery state and follow-up result


  Without 401
  <img width="1917" height="522" alt="happy-path auth tags in Sentry"
  src="https://github.com/user-attachments/assets/3381ed28-8022-43b0-b6c0-623a630e679f"
  />

  ###### Summary
  - Add client-visible 401 diagnostics for auth attachment, upstream auth classification, and 401 request id / cf-ray correlation.
  - Record unauthorized recovery mode, phase, outcome, and retry/follow-up status without changing auth behavior.
  - Surface the highest-signal auth and recovery fields on uploaded client bug reports so they are usable in Sentry.
  - Preserve original unauthorized evidence under `auth_401_*` while keeping follow-up result tags separate.

  ###### Rationale (from spec findings)
  - The dominant bucket needed proof of whether the client attached auth before send or upstream still classified the request as missing auth.
  - Client uploads needed to show whether unauthorized recovery ran and what the client tried next.
  - Request id and cf-ray needed to be preserved on the unauthorized response so server-side correlation is immediate.
  - The bug-report path needed the same auth evidence as the request telemetry path, otherwise the observability would not be operationally useful.

  ###### Scope
  - Add auth 401 and unauthorized-recovery observability in `codex-rs/core`, `codex-rs/codex-api`, and `codex-rs/otel`, including feedback-tag surfacing.
  - Keep auth semantics, refresh behavior, retry behavior, endpoint classification, and geo-denial follow-up work out of this PR.

  ###### Trade-offs
  - This exports only safe auth evidence: header presence/name, upstream auth classification, request ids, and recovery state. It does not export token values or raw upstream bodies.
  - This keeps websocket connection reuse as a transport clue because it can help distinguish stale reused sessions from fresh reconnects.
  - Misroute/base-url classification and geo-denial are intentionally deferred to a separate follow-up PR so this review stays focused on the dominant auth 401 bucket.

  ###### Client follow-up
  - PR 2 will add misroute/provider and geo-denial observability plus the matching feedback-tag surfacing.
  - A separate host/app-server PR should log auth-decision inputs so pre-send host auth state can be correlated with client request evidence.
  - `device_id` remains intentionally separate until there is a safe existing source on the feedback upload path.

  ###### Testing
  - `cargo test -p codex-core refresh_available_models_sorts_by_priority`
  - `cargo test -p codex-core emit_feedback_request_tags_`
  - `cargo test -p codex-core emit_feedback_auth_recovery_tags_`
  - `cargo test -p codex-core auth_request_telemetry_context_tracks_attached_auth_and_retry_phase`
  - `cargo test -p codex-core extract_response_debug_context_decodes_identity_headers`
  - `cargo test -p codex-core identity_auth_details`
  - `cargo test -p codex-core telemetry_error_messages_preserve_non_http_details`
  - `cargo test -p codex-core --all-features --no-run`
  - `cargo test -p codex-otel otel_export_routing_policy_routes_api_request_auth_observability`
  - `cargo test -p codex-otel otel_export_routing_policy_routes_websocket_connect_auth_observability`
  - `cargo test -p codex-otel otel_export_routing_policy_routes_websocket_request_transport_observability`

codex-rs/otel/tests/suite/otel_export_routing_policy.rs

@@ -297,3 +297,462 @@ fn otel_export_routing_policy_routes_tool_result_log_and_trace_events() {
     assert!(!tool_trace_attrs.contains_key("mcp_server"));
     assert!(!tool_trace_attrs.contains_key("mcp_server_origin"));
 }
+
+#[test]
+fn otel_export_routing_policy_routes_auth_recovery_log_and_trace_events() {
+    let log_exporter = InMemoryLogExporter::default();
+    let logger_provider = SdkLoggerProvider::builder()
+        .with_simple_exporter(log_exporter.clone())
+        .build();
+    let span_exporter = InMemorySpanExporter::default();
+    let tracer_provider = SdkTracerProvider::builder()
+        .with_simple_exporter(span_exporter.clone())
+        .build();
+    let tracer = tracer_provider.tracer("sink-split-test");
+
+    let subscriber = tracing_subscriber::registry()
+        .with(
+            opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge::new(
+                &logger_provider,
+            )
+            .with_filter(filter_fn(OtelProvider::log_export_filter)),
+        )
+        .with(
+            tracing_opentelemetry::layer()
+                .with_tracer(tracer)
+                .with_filter(filter_fn(OtelProvider::trace_export_filter)),
+        );
+
+    tracing::subscriber::with_default(subscriber, || {
+        tracing::callsite::rebuild_interest_cache();
+        let manager = SessionTelemetry::new(
+            ThreadId::new(),
+            "gpt-5.1",
+            "gpt-5.1",
+            Some("account-id".to_string()),
+            Some("engineer@example.com".to_string()),
+            Some(TelemetryAuthMode::Chatgpt),
+            "codex_exec".to_string(),
+            true,
+            "tty".to_string(),
+            SessionSource::Cli,
+        );
+        let root_span = tracing::info_span!("root");
+        let _root_guard = root_span.enter();
+        manager.record_auth_recovery(
+            "managed",
+            "reload",
+            "recovery_succeeded",
+            Some("req-401"),
+            Some("ray-401"),
+            Some("missing_authorization_header"),
+            Some("token_expired"),
+            None,
+            Some(true),
+        );
+    });
+
+    logger_provider.force_flush().expect("flush logs");
+    tracer_provider.force_flush().expect("flush traces");
+
+    let logs = log_exporter.get_emitted_logs().expect("log export");
+    let recovery_log = find_log_by_event_name(&logs, "codex.auth_recovery");
+    let recovery_log_attrs = log_attributes(&recovery_log.record);
+    assert_eq!(
+        recovery_log_attrs.get("auth.mode").map(String::as_str),
+        Some("managed")
+    );
+    assert_eq!(
+        recovery_log_attrs.get("auth.step").map(String::as_str),
+        Some("reload")
+    );
+    assert_eq!(
+        recovery_log_attrs.get("auth.outcome").map(String::as_str),
+        Some("recovery_succeeded")
+    );
+    assert_eq!(
+        recovery_log_attrs
+            .get("auth.request_id")
+            .map(String::as_str),
+        Some("req-401")
+    );
+    assert_eq!(
+        recovery_log_attrs.get("auth.cf_ray").map(String::as_str),
+        Some("ray-401")
+    );
+    assert_eq!(
+        recovery_log_attrs.get("auth.error").map(String::as_str),
+        Some("missing_authorization_header")
+    );
+    assert_eq!(
+        recovery_log_attrs
+            .get("auth.error_code")
+            .map(String::as_str),
+        Some("token_expired")
+    );
+    assert_eq!(
+        recovery_log_attrs
+            .get("auth.state_changed")
+            .map(String::as_str),
+        Some("true")
+    );
+
+    let spans = span_exporter.get_finished_spans().expect("span export");
+    assert_eq!(spans.len(), 1);
+    let span_events = &spans[0].events.events;
+    assert_eq!(span_events.len(), 1);
+
+    let recovery_trace_event = find_span_event_by_name_attr(span_events, "codex.auth_recovery");
+    let recovery_trace_attrs = span_event_attributes(recovery_trace_event);
+    assert_eq!(
+        recovery_trace_attrs.get("auth.mode").map(String::as_str),
+        Some("managed")
+    );
+    assert_eq!(
+        recovery_trace_attrs.get("auth.step").map(String::as_str),
+        Some("reload")
+    );
+    assert_eq!(
+        recovery_trace_attrs.get("auth.outcome").map(String::as_str),
+        Some("recovery_succeeded")
+    );
+    assert_eq!(
+        recovery_trace_attrs
+            .get("auth.request_id")
+            .map(String::as_str),
+        Some("req-401")
+    );
+    assert_eq!(
+        recovery_trace_attrs.get("auth.cf_ray").map(String::as_str),
+        Some("ray-401")
+    );
+    assert_eq!(
+        recovery_trace_attrs.get("auth.error").map(String::as_str),
+        Some("missing_authorization_header")
+    );
+    assert_eq!(
+        recovery_trace_attrs
+            .get("auth.error_code")
+            .map(String::as_str),
+        Some("token_expired")
+    );
+    assert_eq!(
+        recovery_trace_attrs
+            .get("auth.state_changed")
+            .map(String::as_str),
+        Some("true")
+    );
+}
+
+#[test]
+fn otel_export_routing_policy_routes_api_request_auth_observability() {
+    let log_exporter = InMemoryLogExporter::default();
+    let logger_provider = SdkLoggerProvider::builder()
+        .with_simple_exporter(log_exporter.clone())
+        .build();
+    let span_exporter = InMemorySpanExporter::default();
+    let tracer_provider = SdkTracerProvider::builder()
+        .with_simple_exporter(span_exporter.clone())
+        .build();
+    let tracer = tracer_provider.tracer("sink-split-test");
+
+    let subscriber = tracing_subscriber::registry()
+        .with(
+            opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge::new(
+                &logger_provider,
+            )
+            .with_filter(filter_fn(OtelProvider::log_export_filter)),
+        )
+        .with(
+            tracing_opentelemetry::layer()
+                .with_tracer(tracer)
+                .with_filter(filter_fn(OtelProvider::trace_export_filter)),
+        );
+
+    tracing::subscriber::with_default(subscriber, || {
+        tracing::callsite::rebuild_interest_cache();
+        let manager = SessionTelemetry::new(
+            ThreadId::new(),
+            "gpt-5.1",
+            "gpt-5.1",
+            Some("account-id".to_string()),
+            Some("engineer@example.com".to_string()),
+            Some(TelemetryAuthMode::Chatgpt),
+            "codex_exec".to_string(),
+            true,
+            "tty".to_string(),
+            SessionSource::Cli,
+        );
+        let root_span = tracing::info_span!("root");
+        let _root_guard = root_span.enter();
+        manager.record_api_request(
+            1,
+            Some(401),
+            Some("http 401"),
+            std::time::Duration::from_millis(42),
+            true,
+            Some("authorization"),
+            true,
+            Some("managed"),
+            Some("refresh_token"),
+            "/responses",
+            Some("req-401"),
+            Some("ray-401"),
+            Some("missing_authorization_header"),
+            Some("token_expired"),
+        );
+    });
+
+    logger_provider.force_flush().expect("flush logs");
+    tracer_provider.force_flush().expect("flush traces");
+
+    let logs = log_exporter.get_emitted_logs().expect("log export");
+    let request_log = find_log_by_event_name(&logs, "codex.api_request");
+    let request_log_attrs = log_attributes(&request_log.record);
+    assert_eq!(
+        request_log_attrs
+            .get("auth.header_attached")
+            .map(String::as_str),
+        Some("true")
+    );
+    assert_eq!(
+        request_log_attrs
+            .get("auth.header_name")
+            .map(String::as_str),
+        Some("authorization")
+    );
+    assert_eq!(
+        request_log_attrs
+            .get("auth.retry_after_unauthorized")
+            .map(String::as_str),
+        Some("true")
+    );
+    assert_eq!(
+        request_log_attrs
+            .get("auth.recovery_mode")
+            .map(String::as_str),
+        Some("managed")
+    );
+    assert_eq!(
+        request_log_attrs
+            .get("auth.recovery_phase")
+            .map(String::as_str),
+        Some("refresh_token")
+    );
+    assert_eq!(
+        request_log_attrs.get("endpoint").map(String::as_str),
+        Some("/responses")
+    );
+    assert_eq!(
+        request_log_attrs.get("auth.error").map(String::as_str),
+        Some("missing_authorization_header")
+    );
+
+    let spans = span_exporter.get_finished_spans().expect("span export");
+    let request_trace_event =
+        find_span_event_by_name_attr(&spans[0].events.events, "codex.api_request");
+    let request_trace_attrs = span_event_attributes(request_trace_event);
+    assert_eq!(
+        request_trace_attrs
+            .get("auth.header_attached")
+            .map(String::as_str),
+        Some("true")
+    );
+    assert_eq!(
+        request_trace_attrs
+            .get("auth.header_name")
+            .map(String::as_str),
+        Some("authorization")
+    );
+    assert_eq!(
+        request_trace_attrs
+            .get("auth.retry_after_unauthorized")
+            .map(String::as_str),
+        Some("true")
+    );
+    assert_eq!(
+        request_trace_attrs.get("endpoint").map(String::as_str),
+        Some("/responses")
+    );
+}
+
+#[test]
+fn otel_export_routing_policy_routes_websocket_connect_auth_observability() {
+    let log_exporter = InMemoryLogExporter::default();
+    let logger_provider = SdkLoggerProvider::builder()
+        .with_simple_exporter(log_exporter.clone())
+        .build();
+    let span_exporter = InMemorySpanExporter::default();
+    let tracer_provider = SdkTracerProvider::builder()
+        .with_simple_exporter(span_exporter.clone())
+        .build();
+    let tracer = tracer_provider.tracer("sink-split-test");
+
+    let subscriber = tracing_subscriber::registry()
+        .with(
+            opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge::new(
+                &logger_provider,
+            )
+            .with_filter(filter_fn(OtelProvider::log_export_filter)),
+        )
+        .with(
+            tracing_opentelemetry::layer()
+                .with_tracer(tracer)
+                .with_filter(filter_fn(OtelProvider::trace_export_filter)),
+        );
+
+    tracing::subscriber::with_default(subscriber, || {
+        tracing::callsite::rebuild_interest_cache();
+        let manager = SessionTelemetry::new(
+            ThreadId::new(),
+            "gpt-5.1",
+            "gpt-5.1",
+            Some("account-id".to_string()),
+            Some("engineer@example.com".to_string()),
+            Some(TelemetryAuthMode::Chatgpt),
+            "codex_exec".to_string(),
+            true,
+            "tty".to_string(),
+            SessionSource::Cli,
+        );
+        let root_span = tracing::info_span!("root");
+        let _root_guard = root_span.enter();
+        manager.record_websocket_connect(
+            std::time::Duration::from_millis(17),
+            Some(401),
+            Some("http 401"),
+            true,
+            Some("authorization"),
+            true,
+            Some("managed"),
+            Some("reload"),
+            "/responses",
+            false,
+            Some("req-ws-401"),
+            Some("ray-ws-401"),
+            Some("missing_authorization_header"),
+            Some("token_expired"),
+        );
+    });
+
+    logger_provider.force_flush().expect("flush logs");
+    tracer_provider.force_flush().expect("flush traces");
+
+    let logs = log_exporter.get_emitted_logs().expect("log export");
+    let connect_log = find_log_by_event_name(&logs, "codex.websocket_connect");
+    let connect_log_attrs = log_attributes(&connect_log.record);
+    assert_eq!(
+        connect_log_attrs
+            .get("auth.header_attached")
+            .map(String::as_str),
+        Some("true")
+    );
+    assert_eq!(
+        connect_log_attrs
+            .get("auth.header_name")
+            .map(String::as_str),
+        Some("authorization")
+    );
+    assert_eq!(
+        connect_log_attrs.get("auth.error").map(String::as_str),
+        Some("missing_authorization_header")
+    );
+    assert_eq!(
+        connect_log_attrs.get("endpoint").map(String::as_str),
+        Some("/responses")
+    );
+    assert_eq!(
+        connect_log_attrs
+            .get("auth.connection_reused")
+            .map(String::as_str),
+        Some("false")
+    );
+
+    let spans = span_exporter.get_finished_spans().expect("span export");
+    let connect_trace_event =
+        find_span_event_by_name_attr(&spans[0].events.events, "codex.websocket_connect");
+    let connect_trace_attrs = span_event_attributes(connect_trace_event);
+    assert_eq!(
+        connect_trace_attrs
+            .get("auth.recovery_phase")
+            .map(String::as_str),
+        Some("reload")
+    );
+}
+
+#[test]
+fn otel_export_routing_policy_routes_websocket_request_transport_observability() {
+    let log_exporter = InMemoryLogExporter::default();
+    let logger_provider = SdkLoggerProvider::builder()
+        .with_simple_exporter(log_exporter.clone())
+        .build();
+    let span_exporter = InMemorySpanExporter::default();
+    let tracer_provider = SdkTracerProvider::builder()
+        .with_simple_exporter(span_exporter.clone())
+        .build();
+    let tracer = tracer_provider.tracer("sink-split-test");
+
+    let subscriber = tracing_subscriber::registry()
+        .with(
+            opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge::new(
+                &logger_provider,
+            )
+            .with_filter(filter_fn(OtelProvider::log_export_filter)),
+        )
+        .with(
+            tracing_opentelemetry::layer()
+                .with_tracer(tracer)
+                .with_filter(filter_fn(OtelProvider::trace_export_filter)),
+        );
+
+    tracing::subscriber::with_default(subscriber, || {
+        tracing::callsite::rebuild_interest_cache();
+        let manager = SessionTelemetry::new(
+            ThreadId::new(),
+            "gpt-5.1",
+            "gpt-5.1",
+            Some("account-id".to_string()),
+            Some("engineer@example.com".to_string()),
+            Some(TelemetryAuthMode::Chatgpt),
+            "codex_exec".to_string(),
+            true,
+            "tty".to_string(),
+            SessionSource::Cli,
+        );
+        let root_span = tracing::info_span!("root");
+        let _root_guard = root_span.enter();
+        manager.record_websocket_request(
+            std::time::Duration::from_millis(23),
+            Some("stream error"),
+            true,
+        );
+    });
+
+    logger_provider.force_flush().expect("flush logs");
+    tracer_provider.force_flush().expect("flush traces");
+
+    let logs = log_exporter.get_emitted_logs().expect("log export");
+    let request_log = find_log_by_event_name(&logs, "codex.websocket_request");
+    let request_log_attrs = log_attributes(&request_log.record);
+    assert_eq!(
+        request_log_attrs
+            .get("auth.connection_reused")
+            .map(String::as_str),
+        Some("true")
+    );
+    assert_eq!(
+        request_log_attrs.get("error.message").map(String::as_str),
+        Some("stream error")
+    );
+
+    let spans = span_exporter.get_finished_spans().expect("span export");
+    let request_trace_event =
+        find_span_event_by_name_attr(&spans[0].events.events, "codex.websocket_request");
+    let request_trace_attrs = span_event_attributes(request_trace_event);
+    assert_eq!(
+        request_trace_attrs
+            .get("auth.connection_reused")
+            .map(String::as_str),
+        Some("true")
+    );
+}

codex-rs/otel/tests/suite/runtime_summary.rs

@@ -47,8 +47,23 @@ fn runtime_metrics_summary_collects_tool_api_and_streaming_metrics() -> Result<(
         None,
         None,
     );
-    manager.record_api_request(1, Some(200), None, Duration::from_millis(300));
-    manager.record_websocket_request(Duration::from_millis(400), None);
+    manager.record_api_request(
+        1,
+        Some(200),
+        None,
+        Duration::from_millis(300),
+        false,
+        None,
+        false,
+        None,
+        None,
+        "/responses",
+        None,
+        None,
+        None,
+        None,
+    );
+    manager.record_websocket_request(Duration::from_millis(400), None, false);
     let sse_response: std::result::Result<
         Option<std::result::Result<StreamEvent, eventsource_stream::EventStreamError<&str>>>,
         tokio::time::error::Elapsed,