mirror of
https://github.com/openai/codex.git
synced 2026-04-09 00:51:43 +03:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2f44170063 | ||
|
|
32a0c52acd |
@@ -30,6 +30,7 @@ use crate::facts::SubAgentThreadStartedInput;
|
||||
use crate::facts::TrackEventsContext;
|
||||
use crate::facts::TurnResolvedConfigFact;
|
||||
use crate::facts::TurnStatus;
|
||||
use crate::facts::TurnTokenUsageFact;
|
||||
use crate::reducer::AnalyticsReducer;
|
||||
use crate::reducer::normalize_path_for_skill_id;
|
||||
use crate::reducer::skill_id_for_local_skill;
|
||||
@@ -38,8 +39,11 @@ use codex_app_server_protocol::AskForApproval as AppServerAskForApproval;
|
||||
use codex_app_server_protocol::ClientInfo;
|
||||
use codex_app_server_protocol::ClientRequest;
|
||||
use codex_app_server_protocol::ClientResponse;
|
||||
use codex_app_server_protocol::CodexErrorInfo;
|
||||
use codex_app_server_protocol::InitializeCapabilities;
|
||||
use codex_app_server_protocol::InitializeParams;
|
||||
use codex_app_server_protocol::JSONRPCErrorError;
|
||||
use codex_app_server_protocol::NonSteerableTurnKind;
|
||||
use codex_app_server_protocol::RequestId;
|
||||
use codex_app_server_protocol::SandboxPolicy as AppServerSandboxPolicy;
|
||||
use codex_app_server_protocol::ServerNotification;
|
||||
@@ -54,6 +58,8 @@ use codex_app_server_protocol::TurnError as AppServerTurnError;
|
||||
use codex_app_server_protocol::TurnStartParams;
|
||||
use codex_app_server_protocol::TurnStartedNotification;
|
||||
use codex_app_server_protocol::TurnStatus as AppServerTurnStatus;
|
||||
use codex_app_server_protocol::TurnSteerParams;
|
||||
use codex_app_server_protocol::TurnSteerResponse;
|
||||
use codex_app_server_protocol::UserInput;
|
||||
use codex_login::default_client::DEFAULT_ORIGINATOR;
|
||||
use codex_login::default_client::originator;
|
||||
@@ -66,6 +72,7 @@ use codex_protocol::config_types::ModeKind;
|
||||
use codex_protocol::protocol::AskForApproval;
|
||||
use codex_protocol::protocol::SandboxPolicy;
|
||||
use codex_protocol::protocol::SubAgentSource;
|
||||
use codex_protocol::protocol::TokenUsage;
|
||||
use pretty_assertions::assert_eq;
|
||||
use serde_json::json;
|
||||
use std::collections::HashSet;
|
||||
@@ -181,6 +188,20 @@ fn sample_turn_started_notification(thread_id: &str, turn_id: &str) -> ServerNot
|
||||
})
|
||||
}
|
||||
|
||||
fn sample_turn_token_usage_fact(thread_id: &str, turn_id: &str) -> TurnTokenUsageFact {
|
||||
TurnTokenUsageFact {
|
||||
thread_id: thread_id.to_string(),
|
||||
turn_id: turn_id.to_string(),
|
||||
token_usage: TokenUsage {
|
||||
total_tokens: 321,
|
||||
input_tokens: 123,
|
||||
cached_input_tokens: 45,
|
||||
output_tokens: 140,
|
||||
reasoning_output_tokens: 13,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_turn_completed_notification(
|
||||
thread_id: &str,
|
||||
turn_id: &str,
|
||||
@@ -226,12 +247,91 @@ fn sample_turn_resolved_config(turn_id: &str) -> TurnResolvedConfigFact {
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_runtime_metadata() -> CodexRuntimeMetadata {
|
||||
CodexRuntimeMetadata {
|
||||
codex_rs_version: "0.1.0".to_string(),
|
||||
runtime_os: "macos".to_string(),
|
||||
runtime_os_version: "15.3.1".to_string(),
|
||||
runtime_arch: "aarch64".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_turn_steer_request(
|
||||
thread_id: &str,
|
||||
expected_turn_id: &str,
|
||||
request_id: i64,
|
||||
) -> ClientRequest {
|
||||
ClientRequest::TurnSteer {
|
||||
request_id: RequestId::Integer(request_id),
|
||||
params: TurnSteerParams {
|
||||
thread_id: thread_id.to_string(),
|
||||
expected_turn_id: expected_turn_id.to_string(),
|
||||
input: vec![
|
||||
UserInput::Text {
|
||||
text: "more".to_string(),
|
||||
text_elements: vec![],
|
||||
},
|
||||
UserInput::LocalImage {
|
||||
path: "/tmp/a.png".into(),
|
||||
},
|
||||
],
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_turn_steer_response(turn_id: &str, request_id: i64) -> ClientResponse {
|
||||
ClientResponse::TurnSteer {
|
||||
request_id: RequestId::Integer(request_id),
|
||||
response: TurnSteerResponse {
|
||||
turn_id: turn_id.to_string(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn no_active_turn_steer_error() -> JSONRPCErrorError {
|
||||
JSONRPCErrorError {
|
||||
code: -32600,
|
||||
message: "no active turn to steer".to_string(),
|
||||
data: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn non_steerable_review_error() -> JSONRPCErrorError {
|
||||
JSONRPCErrorError {
|
||||
code: -32600,
|
||||
message: "cannot steer a review turn".to_string(),
|
||||
data: Some(
|
||||
serde_json::to_value(AppServerTurnError {
|
||||
message: "cannot steer a review turn".to_string(),
|
||||
codex_error_info: Some(CodexErrorInfo::ActiveTurnNotSteerable {
|
||||
turn_kind: NonSteerableTurnKind::Review,
|
||||
}),
|
||||
additional_details: None,
|
||||
})
|
||||
.expect("serialize turn error"),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
fn input_too_large_steer_error() -> JSONRPCErrorError {
|
||||
JSONRPCErrorError {
|
||||
code: -32602,
|
||||
message: "Input exceeds the maximum length of 1048576 characters.".to_string(),
|
||||
data: Some(json!({
|
||||
"input_error_code": "input_too_large",
|
||||
"actual_chars": 1048577,
|
||||
"max_chars": 1048576,
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
async fn ingest_turn_prerequisites(
|
||||
reducer: &mut AnalyticsReducer,
|
||||
out: &mut Vec<TrackEventRequest>,
|
||||
include_initialize: bool,
|
||||
include_resolved_config: bool,
|
||||
include_started: bool,
|
||||
include_token_usage: bool,
|
||||
) {
|
||||
if include_initialize {
|
||||
reducer
|
||||
@@ -301,6 +401,17 @@ async fn ingest_turn_prerequisites(
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
if include_token_usage {
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Custom(CustomAnalyticsFact::TurnTokenUsage(Box::new(
|
||||
sample_turn_token_usage_fact("thread-2", "turn-2"),
|
||||
))),
|
||||
out,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
fn expected_absolute_path(path: &PathBuf) -> String {
|
||||
@@ -1036,7 +1147,7 @@ fn turn_event_serializes_expected_shape() {
|
||||
is_first_turn: true,
|
||||
status: Some(TurnStatus::Completed),
|
||||
turn_error: None,
|
||||
steer_count: None,
|
||||
steer_count: Some(0),
|
||||
total_tool_call_count: None,
|
||||
shell_command_count: None,
|
||||
file_change_count: None,
|
||||
@@ -1045,6 +1156,11 @@ fn turn_event_serializes_expected_shape() {
|
||||
subagent_tool_call_count: None,
|
||||
web_search_count: None,
|
||||
image_generation_count: None,
|
||||
input_tokens: None,
|
||||
cached_input_tokens: None,
|
||||
output_tokens: None,
|
||||
reasoning_output_tokens: None,
|
||||
total_tokens: None,
|
||||
duration_ms: Some(1234),
|
||||
started_at: Some(455),
|
||||
completed_at: Some(456),
|
||||
@@ -1077,7 +1193,7 @@ fn turn_event_serializes_expected_shape() {
|
||||
"is_first_turn": true,
|
||||
"status": "completed",
|
||||
"turn_error": null,
|
||||
"steer_count": null,
|
||||
"steer_count": 0,
|
||||
"total_tool_call_count": null,
|
||||
"shell_command_count": null,
|
||||
"file_change_count": null,
|
||||
@@ -1086,6 +1202,11 @@ fn turn_event_serializes_expected_shape() {
|
||||
"subagent_tool_call_count": null,
|
||||
"web_search_count": null,
|
||||
"image_generation_count": null,
|
||||
"input_tokens": null,
|
||||
"cached_input_tokens": null,
|
||||
"output_tokens": null,
|
||||
"reasoning_output_tokens": null,
|
||||
"total_tokens": null,
|
||||
"duration_ms": 1234,
|
||||
"started_at": 455,
|
||||
"completed_at": 456
|
||||
@@ -1094,6 +1215,265 @@ fn turn_event_serializes_expected_shape() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accepted_turn_steer_emits_expected_event() {
|
||||
let mut reducer = AnalyticsReducer::default();
|
||||
let mut out = Vec::new();
|
||||
|
||||
ingest_turn_prerequisites(
|
||||
&mut reducer,
|
||||
&mut out,
|
||||
/*include_initialize*/ true,
|
||||
/*include_resolved_config*/ false,
|
||||
/*include_started*/ false,
|
||||
/*include_token_usage*/ false,
|
||||
)
|
||||
.await;
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Request {
|
||||
connection_id: 7,
|
||||
request_id: RequestId::Integer(4),
|
||||
request: Box::new(sample_turn_steer_request(
|
||||
"thread-2", "turn-2", /*request_id*/ 4,
|
||||
)),
|
||||
},
|
||||
&mut out,
|
||||
)
|
||||
.await;
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Response {
|
||||
connection_id: 7,
|
||||
response: Box::new(sample_turn_steer_response("turn-2", /*request_id*/ 4)),
|
||||
},
|
||||
&mut out,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(out.len(), 1);
|
||||
let payload = serde_json::to_value(&out[0]).expect("serialize turn steer event");
|
||||
assert_eq!(payload["event_type"], json!("codex_turn_steer_event"));
|
||||
assert_eq!(payload["event_params"]["thread_id"], json!("thread-2"));
|
||||
assert_eq!(payload["event_params"]["expected_turn_id"], json!("turn-2"));
|
||||
assert_eq!(payload["event_params"]["accepted_turn_id"], json!("turn-2"));
|
||||
assert_eq!(payload["event_params"]["num_input_images"], json!(1));
|
||||
assert_eq!(payload["event_params"]["result"], json!("accepted"));
|
||||
assert_eq!(payload["event_params"]["rejection_reason"], json!(null));
|
||||
assert!(
|
||||
payload["event_params"]["created_at"]
|
||||
.as_u64()
|
||||
.expect("created_at")
|
||||
> 0
|
||||
);
|
||||
assert_eq!(
|
||||
payload["event_params"]["app_server_client"]["product_client_id"],
|
||||
json!("codex-tui")
|
||||
);
|
||||
assert_eq!(
|
||||
payload["event_params"]["runtime"]["codex_rs_version"],
|
||||
json!("0.1.0")
|
||||
);
|
||||
assert!(payload["event_params"].get("product_client_id").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejected_turn_steer_uses_request_connection_metadata() {
|
||||
let mut reducer = AnalyticsReducer::default();
|
||||
let mut out = Vec::new();
|
||||
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Initialize {
|
||||
connection_id: 7,
|
||||
params: InitializeParams {
|
||||
client_info: ClientInfo {
|
||||
name: "codex-tui".to_string(),
|
||||
title: None,
|
||||
version: "1.0.0".to_string(),
|
||||
},
|
||||
capabilities: None,
|
||||
},
|
||||
product_client_id: "codex-tui".to_string(),
|
||||
runtime: sample_runtime_metadata(),
|
||||
rpc_transport: AppServerRpcTransport::Stdio,
|
||||
},
|
||||
&mut out,
|
||||
)
|
||||
.await;
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Response {
|
||||
connection_id: 7,
|
||||
response: Box::new(sample_thread_start_response(
|
||||
"thread-2", /*ephemeral*/ false, "gpt-5",
|
||||
)),
|
||||
},
|
||||
&mut out,
|
||||
)
|
||||
.await;
|
||||
out.clear();
|
||||
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Request {
|
||||
connection_id: 7,
|
||||
request_id: RequestId::Integer(4),
|
||||
request: Box::new(sample_turn_steer_request(
|
||||
"thread-2", "turn-2", /*request_id*/ 4,
|
||||
)),
|
||||
},
|
||||
&mut out,
|
||||
)
|
||||
.await;
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::ErrorResponse {
|
||||
connection_id: 7,
|
||||
request_id: RequestId::Integer(4),
|
||||
error: no_active_turn_steer_error(),
|
||||
},
|
||||
&mut out,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(out.len(), 1);
|
||||
let payload = serde_json::to_value(&out[0]).expect("serialize turn steer event");
|
||||
assert_eq!(payload["event_type"], json!("codex_turn_steer_event"));
|
||||
assert_eq!(payload["event_params"]["thread_id"], json!("thread-2"));
|
||||
assert_eq!(payload["event_params"]["expected_turn_id"], json!("turn-2"));
|
||||
assert_eq!(payload["event_params"]["accepted_turn_id"], json!(null));
|
||||
assert_eq!(payload["event_params"]["num_input_images"], json!(1));
|
||||
assert_eq!(
|
||||
payload["event_params"]["app_server_client"]["product_client_id"],
|
||||
json!("codex-tui")
|
||||
);
|
||||
assert_eq!(
|
||||
payload["event_params"]["runtime"]["codex_rs_version"],
|
||||
json!("0.1.0")
|
||||
);
|
||||
assert_eq!(payload["event_params"]["result"], json!("rejected"));
|
||||
assert_eq!(
|
||||
payload["event_params"]["rejection_reason"],
|
||||
json!("no_active_turn")
|
||||
);
|
||||
assert!(
|
||||
payload["event_params"]["created_at"]
|
||||
.as_u64()
|
||||
.expect("created_at")
|
||||
> 0
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejected_turn_steer_maps_active_turn_not_steerable_error_data() {
|
||||
let mut reducer = AnalyticsReducer::default();
|
||||
let mut out = Vec::new();
|
||||
|
||||
ingest_turn_prerequisites(
|
||||
&mut reducer,
|
||||
&mut out,
|
||||
/*include_initialize*/ true,
|
||||
/*include_resolved_config*/ false,
|
||||
/*include_started*/ false,
|
||||
/*include_token_usage*/ false,
|
||||
)
|
||||
.await;
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Request {
|
||||
connection_id: 7,
|
||||
request_id: RequestId::Integer(4),
|
||||
request: Box::new(sample_turn_steer_request(
|
||||
"thread-2", "turn-2", /*request_id*/ 4,
|
||||
)),
|
||||
},
|
||||
&mut out,
|
||||
)
|
||||
.await;
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::ErrorResponse {
|
||||
connection_id: 7,
|
||||
request_id: RequestId::Integer(4),
|
||||
error: non_steerable_review_error(),
|
||||
},
|
||||
&mut out,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(out.len(), 1);
|
||||
let payload = serde_json::to_value(&out[0]).expect("serialize turn steer event");
|
||||
assert_eq!(
|
||||
payload["event_params"]["rejection_reason"],
|
||||
json!("non_steerable_review")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejected_turn_steer_maps_input_too_large_error_data() {
|
||||
let mut reducer = AnalyticsReducer::default();
|
||||
let mut out = Vec::new();
|
||||
|
||||
ingest_turn_prerequisites(
|
||||
&mut reducer,
|
||||
&mut out,
|
||||
/*include_initialize*/ true,
|
||||
/*include_resolved_config*/ false,
|
||||
/*include_started*/ false,
|
||||
/*include_token_usage*/ false,
|
||||
)
|
||||
.await;
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Request {
|
||||
connection_id: 7,
|
||||
request_id: RequestId::Integer(4),
|
||||
request: Box::new(sample_turn_steer_request(
|
||||
"thread-2", "turn-2", /*request_id*/ 4,
|
||||
)),
|
||||
},
|
||||
&mut out,
|
||||
)
|
||||
.await;
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::ErrorResponse {
|
||||
connection_id: 7,
|
||||
request_id: RequestId::Integer(4),
|
||||
error: input_too_large_steer_error(),
|
||||
},
|
||||
&mut out,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(out.len(), 1);
|
||||
let payload = serde_json::to_value(&out[0]).expect("serialize turn steer event");
|
||||
assert_eq!(
|
||||
payload["event_params"]["rejection_reason"],
|
||||
json!("input_too_large")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn turn_steer_does_not_emit_without_pending_request() {
|
||||
let mut reducer = AnalyticsReducer::default();
|
||||
let mut out = Vec::new();
|
||||
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::ErrorResponse {
|
||||
connection_id: 7,
|
||||
request_id: RequestId::Integer(4),
|
||||
error: no_active_turn_steer_error(),
|
||||
},
|
||||
&mut out,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(out.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn turn_lifecycle_emits_turn_event() {
|
||||
let mut reducer = AnalyticsReducer::default();
|
||||
@@ -1105,6 +1485,7 @@ async fn turn_lifecycle_emits_turn_event() {
|
||||
/*include_initialize*/ true,
|
||||
/*include_resolved_config*/ true,
|
||||
/*include_started*/ true,
|
||||
/*include_token_usage*/ true,
|
||||
)
|
||||
.await;
|
||||
reducer
|
||||
@@ -1130,9 +1511,120 @@ async fn turn_lifecycle_emits_turn_event() {
|
||||
);
|
||||
assert_eq!(payload["event_params"]["num_input_images"], json!(1));
|
||||
assert_eq!(payload["event_params"]["status"], json!("completed"));
|
||||
assert_eq!(payload["event_params"]["steer_count"], json!(0));
|
||||
assert_eq!(payload["event_params"]["started_at"], json!(455));
|
||||
assert_eq!(payload["event_params"]["completed_at"], json!(456));
|
||||
assert_eq!(payload["event_params"]["duration_ms"], json!(1234));
|
||||
assert_eq!(payload["event_params"]["input_tokens"], json!(123));
|
||||
assert_eq!(payload["event_params"]["cached_input_tokens"], json!(45));
|
||||
assert_eq!(payload["event_params"]["output_tokens"], json!(140));
|
||||
assert_eq!(
|
||||
payload["event_params"]["reasoning_output_tokens"],
|
||||
json!(13)
|
||||
);
|
||||
assert_eq!(payload["event_params"]["total_tokens"], json!(321));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accepted_steers_increment_turn_steer_count() {
|
||||
let mut reducer = AnalyticsReducer::default();
|
||||
let mut out = Vec::new();
|
||||
|
||||
ingest_turn_prerequisites(
|
||||
&mut reducer,
|
||||
&mut out,
|
||||
/*include_initialize*/ true,
|
||||
/*include_resolved_config*/ true,
|
||||
/*include_started*/ true,
|
||||
/*include_token_usage*/ false,
|
||||
)
|
||||
.await;
|
||||
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Request {
|
||||
connection_id: 7,
|
||||
request_id: RequestId::Integer(4),
|
||||
request: Box::new(sample_turn_steer_request(
|
||||
"thread-2", "turn-2", /*request_id*/ 4,
|
||||
)),
|
||||
},
|
||||
&mut out,
|
||||
)
|
||||
.await;
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Response {
|
||||
connection_id: 7,
|
||||
response: Box::new(sample_turn_steer_response("turn-2", /*request_id*/ 4)),
|
||||
},
|
||||
&mut out,
|
||||
)
|
||||
.await;
|
||||
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Request {
|
||||
connection_id: 7,
|
||||
request_id: RequestId::Integer(5),
|
||||
request: Box::new(sample_turn_steer_request(
|
||||
"thread-2", "turn-2", /*request_id*/ 5,
|
||||
)),
|
||||
},
|
||||
&mut out,
|
||||
)
|
||||
.await;
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::ErrorResponse {
|
||||
connection_id: 7,
|
||||
request_id: RequestId::Integer(5),
|
||||
error: no_active_turn_steer_error(),
|
||||
},
|
||||
&mut out,
|
||||
)
|
||||
.await;
|
||||
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Request {
|
||||
connection_id: 7,
|
||||
request_id: RequestId::Integer(6),
|
||||
request: Box::new(sample_turn_steer_request(
|
||||
"thread-2", "turn-2", /*request_id*/ 6,
|
||||
)),
|
||||
},
|
||||
&mut out,
|
||||
)
|
||||
.await;
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Response {
|
||||
connection_id: 7,
|
||||
response: Box::new(sample_turn_steer_response("turn-2", /*request_id*/ 6)),
|
||||
},
|
||||
&mut out,
|
||||
)
|
||||
.await;
|
||||
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Notification(Box::new(sample_turn_completed_notification(
|
||||
"thread-2",
|
||||
"turn-2",
|
||||
AppServerTurnStatus::Completed,
|
||||
/*codex_error_info*/ None,
|
||||
))),
|
||||
&mut out,
|
||||
)
|
||||
.await;
|
||||
|
||||
let turn_event = out
|
||||
.iter()
|
||||
.find(|event| matches!(event, TrackEventRequest::TurnEvent(_)))
|
||||
.expect("turn event should be emitted");
|
||||
let payload = serde_json::to_value(turn_event).expect("serialize turn event");
|
||||
assert_eq!(payload["event_params"]["steer_count"], json!(2));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1146,6 +1638,7 @@ async fn turn_does_not_emit_without_required_prerequisites() {
|
||||
/*include_initialize*/ false,
|
||||
/*include_resolved_config*/ true,
|
||||
/*include_started*/ false,
|
||||
/*include_token_usage*/ false,
|
||||
)
|
||||
.await;
|
||||
reducer
|
||||
@@ -1175,6 +1668,7 @@ async fn turn_does_not_emit_without_required_prerequisites() {
|
||||
/*include_initialize*/ true,
|
||||
/*include_resolved_config*/ false,
|
||||
/*include_started*/ false,
|
||||
/*include_token_usage*/ false,
|
||||
)
|
||||
.await;
|
||||
reducer
|
||||
@@ -1202,6 +1696,7 @@ async fn turn_lifecycle_emits_failed_turn_event() {
|
||||
/*include_initialize*/ true,
|
||||
/*include_resolved_config*/ true,
|
||||
/*include_started*/ true,
|
||||
/*include_token_usage*/ false,
|
||||
)
|
||||
.await;
|
||||
reducer
|
||||
@@ -1233,6 +1728,7 @@ async fn turn_lifecycle_emits_interrupted_turn_event_without_error() {
|
||||
/*include_initialize*/ true,
|
||||
/*include_resolved_config*/ true,
|
||||
/*include_started*/ true,
|
||||
/*include_token_usage*/ false,
|
||||
)
|
||||
.await;
|
||||
reducer
|
||||
@@ -1264,6 +1760,7 @@ async fn turn_completed_without_started_notification_emits_null_started_at() {
|
||||
/*include_initialize*/ true,
|
||||
/*include_resolved_config*/ true,
|
||||
/*include_started*/ false,
|
||||
/*include_token_usage*/ false,
|
||||
)
|
||||
.await;
|
||||
reducer
|
||||
@@ -1281,6 +1778,14 @@ async fn turn_completed_without_started_notification_emits_null_started_at() {
|
||||
let payload = serde_json::to_value(&out[0]).expect("serialize turn event");
|
||||
assert_eq!(payload["event_params"]["started_at"], json!(null));
|
||||
assert_eq!(payload["event_params"]["duration_ms"], json!(1234));
|
||||
assert_eq!(payload["event_params"]["input_tokens"], json!(null));
|
||||
assert_eq!(payload["event_params"]["cached_input_tokens"], json!(null));
|
||||
assert_eq!(payload["event_params"]["output_tokens"], json!(null));
|
||||
assert_eq!(
|
||||
payload["event_params"]["reasoning_output_tokens"],
|
||||
json!(null)
|
||||
);
|
||||
assert_eq!(payload["event_params"]["total_tokens"], json!(null));
|
||||
}
|
||||
|
||||
fn sample_plugin_metadata() -> PluginTelemetryMetadata {
|
||||
|
||||
@@ -14,10 +14,12 @@ use crate::facts::SkillInvokedInput;
|
||||
use crate::facts::SubAgentThreadStartedInput;
|
||||
use crate::facts::TrackEventsContext;
|
||||
use crate::facts::TurnResolvedConfigFact;
|
||||
use crate::facts::TurnTokenUsageFact;
|
||||
use crate::reducer::AnalyticsReducer;
|
||||
use codex_app_server_protocol::ClientRequest;
|
||||
use codex_app_server_protocol::ClientResponse;
|
||||
use codex_app_server_protocol::InitializeParams;
|
||||
use codex_app_server_protocol::JSONRPCErrorError;
|
||||
use codex_app_server_protocol::RequestId;
|
||||
use codex_app_server_protocol::ServerNotification;
|
||||
use codex_login::AuthManager;
|
||||
@@ -196,6 +198,12 @@ impl AnalyticsEventsClient {
|
||||
));
|
||||
}
|
||||
|
||||
pub fn track_turn_token_usage(&self, fact: TurnTokenUsageFact) {
|
||||
self.record_fact(AnalyticsFact::Custom(CustomAnalyticsFact::TurnTokenUsage(
|
||||
Box::new(fact),
|
||||
)));
|
||||
}
|
||||
|
||||
pub fn track_plugin_installed(&self, plugin: PluginTelemetryMetadata) {
|
||||
self.record_fact(AnalyticsFact::Custom(
|
||||
CustomAnalyticsFact::PluginStateChanged(PluginStateChangedInput {
|
||||
@@ -246,6 +254,19 @@ impl AnalyticsEventsClient {
|
||||
});
|
||||
}
|
||||
|
||||
pub fn track_error_response(
|
||||
&self,
|
||||
connection_id: u64,
|
||||
request_id: RequestId,
|
||||
error: JSONRPCErrorError,
|
||||
) {
|
||||
self.record_fact(AnalyticsFact::ErrorResponse {
|
||||
connection_id,
|
||||
request_id,
|
||||
error,
|
||||
});
|
||||
}
|
||||
|
||||
pub fn track_notification(&self, notification: ServerNotification) {
|
||||
self.record_fact(AnalyticsFact::Notification(Box::new(notification)));
|
||||
}
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
use crate::facts::AppInvocation;
|
||||
use crate::facts::CodexTurnSteerEvent;
|
||||
use crate::facts::InvocationType;
|
||||
use crate::facts::PluginState;
|
||||
use crate::facts::SubAgentThreadStartedInput;
|
||||
use crate::facts::TrackEventsContext;
|
||||
use crate::facts::TurnStatus;
|
||||
use crate::facts::TurnSteerRejectionReason;
|
||||
use crate::facts::TurnSteerResult;
|
||||
use crate::facts::TurnSubmissionType;
|
||||
use codex_app_server_protocol::CodexErrorInfo;
|
||||
use codex_login::default_client::originator;
|
||||
@@ -41,6 +44,7 @@ pub(crate) enum TrackEventRequest {
|
||||
AppMentioned(CodexAppMentionedEventRequest),
|
||||
AppUsed(CodexAppUsedEventRequest),
|
||||
TurnEvent(Box<CodexTurnEventRequest>),
|
||||
TurnSteer(CodexTurnSteerEventRequest),
|
||||
PluginUsed(CodexPluginUsedEventRequest),
|
||||
PluginInstalled(CodexPluginEventRequest),
|
||||
PluginUninstalled(CodexPluginEventRequest),
|
||||
@@ -156,6 +160,11 @@ pub(crate) struct CodexTurnEventParams {
|
||||
pub(crate) subagent_tool_call_count: Option<usize>,
|
||||
pub(crate) web_search_count: Option<usize>,
|
||||
pub(crate) image_generation_count: Option<usize>,
|
||||
pub(crate) input_tokens: Option<i64>,
|
||||
pub(crate) cached_input_tokens: Option<i64>,
|
||||
pub(crate) output_tokens: Option<i64>,
|
||||
pub(crate) reasoning_output_tokens: Option<i64>,
|
||||
pub(crate) total_tokens: Option<i64>,
|
||||
pub(crate) duration_ms: Option<u64>,
|
||||
pub(crate) started_at: Option<u64>,
|
||||
pub(crate) completed_at: Option<u64>,
|
||||
@@ -167,6 +176,25 @@ pub(crate) struct CodexTurnEventRequest {
|
||||
pub(crate) event_params: CodexTurnEventParams,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(crate) struct CodexTurnSteerEventParams {
|
||||
pub(crate) thread_id: String,
|
||||
pub(crate) expected_turn_id: Option<String>,
|
||||
pub(crate) accepted_turn_id: Option<String>,
|
||||
pub(crate) app_server_client: CodexAppServerClientMetadata,
|
||||
pub(crate) runtime: CodexRuntimeMetadata,
|
||||
pub(crate) num_input_images: usize,
|
||||
pub(crate) result: TurnSteerResult,
|
||||
pub(crate) rejection_reason: Option<TurnSteerRejectionReason>,
|
||||
pub(crate) created_at: u64,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(crate) struct CodexTurnSteerEventRequest {
|
||||
pub(crate) event_type: &'static str,
|
||||
pub(crate) event_params: CodexTurnSteerEventParams,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(crate) struct CodexPluginMetadata {
|
||||
pub(crate) plugin_id: Option<String>,
|
||||
@@ -258,6 +286,25 @@ pub(crate) fn codex_plugin_used_metadata(
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn codex_turn_steer_event_params(
|
||||
app_server_client: CodexAppServerClientMetadata,
|
||||
runtime: CodexRuntimeMetadata,
|
||||
tracking: &TrackEventsContext,
|
||||
turn_steer: CodexTurnSteerEvent,
|
||||
) -> CodexTurnSteerEventParams {
|
||||
CodexTurnSteerEventParams {
|
||||
thread_id: tracking.thread_id.clone(),
|
||||
expected_turn_id: turn_steer.expected_turn_id,
|
||||
accepted_turn_id: turn_steer.accepted_turn_id,
|
||||
app_server_client,
|
||||
runtime,
|
||||
num_input_images: turn_steer.num_input_images,
|
||||
result: turn_steer.result,
|
||||
rejection_reason: turn_steer.rejection_reason,
|
||||
created_at: turn_steer.created_at,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn thread_source_name(thread_source: &SessionSource) -> Option<&'static str> {
|
||||
match thread_source {
|
||||
SessionSource::Cli | SessionSource::VSCode | SessionSource::Exec => Some("user"),
|
||||
|
||||
@@ -3,6 +3,7 @@ use crate::events::CodexRuntimeMetadata;
|
||||
use codex_app_server_protocol::ClientRequest;
|
||||
use codex_app_server_protocol::ClientResponse;
|
||||
use codex_app_server_protocol::InitializeParams;
|
||||
use codex_app_server_protocol::JSONRPCErrorError;
|
||||
use codex_app_server_protocol::RequestId;
|
||||
use codex_app_server_protocol::ServerNotification;
|
||||
use codex_plugin::PluginTelemetryMetadata;
|
||||
@@ -16,6 +17,7 @@ use codex_protocol::protocol::AskForApproval;
|
||||
use codex_protocol::protocol::SandboxPolicy;
|
||||
use codex_protocol::protocol::SkillScope;
|
||||
use codex_protocol::protocol::SubAgentSource;
|
||||
use codex_protocol::protocol::TokenUsage;
|
||||
use serde::Serialize;
|
||||
use std::path::PathBuf;
|
||||
|
||||
@@ -65,6 +67,13 @@ pub struct TurnResolvedConfigFact {
|
||||
pub is_first_turn: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct TurnTokenUsageFact {
|
||||
pub turn_id: String,
|
||||
pub thread_id: String,
|
||||
pub token_usage: TokenUsage,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum TurnStatus {
|
||||
@@ -73,6 +82,34 @@ pub enum TurnStatus {
|
||||
Interrupted,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum TurnSteerResult {
|
||||
Accepted,
|
||||
Rejected,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum TurnSteerRejectionReason {
|
||||
NoActiveTurn,
|
||||
ExpectedTurnMismatch,
|
||||
NonSteerableReview,
|
||||
NonSteerableCompact,
|
||||
EmptyInput,
|
||||
InputTooLarge,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct CodexTurnSteerEvent {
|
||||
pub expected_turn_id: Option<String>,
|
||||
pub accepted_turn_id: Option<String>,
|
||||
pub num_input_images: usize,
|
||||
pub result: TurnSteerResult,
|
||||
pub rejection_reason: Option<TurnSteerRejectionReason>,
|
||||
pub created_at: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct SkillInvocation {
|
||||
pub skill_name: String,
|
||||
@@ -124,6 +161,11 @@ pub(crate) enum AnalyticsFact {
|
||||
connection_id: u64,
|
||||
response: Box<ClientResponse>,
|
||||
},
|
||||
ErrorResponse {
|
||||
connection_id: u64,
|
||||
request_id: RequestId,
|
||||
error: JSONRPCErrorError,
|
||||
},
|
||||
Notification(Box<ServerNotification>),
|
||||
// Facts that do not naturally exist on the app-server protocol surface, or
|
||||
// would require non-trivial protocol reshaping on this branch.
|
||||
@@ -133,6 +175,7 @@ pub(crate) enum AnalyticsFact {
|
||||
pub(crate) enum CustomAnalyticsFact {
|
||||
SubAgentThreadStarted(SubAgentThreadStartedInput),
|
||||
TurnResolvedConfig(Box<TurnResolvedConfigFact>),
|
||||
TurnTokenUsage(Box<TurnTokenUsageFact>),
|
||||
SkillInvoked(SkillInvokedInput),
|
||||
AppMentioned(AppMentionedInput),
|
||||
AppUsed(AppUsedInput),
|
||||
|
||||
@@ -6,12 +6,16 @@ mod reducer;
|
||||
pub use client::AnalyticsEventsClient;
|
||||
pub use events::AppServerRpcTransport;
|
||||
pub use facts::AppInvocation;
|
||||
pub use facts::CodexTurnSteerEvent;
|
||||
pub use facts::InvocationType;
|
||||
pub use facts::SkillInvocation;
|
||||
pub use facts::SubAgentThreadStartedInput;
|
||||
pub use facts::TrackEventsContext;
|
||||
pub use facts::TurnResolvedConfigFact;
|
||||
pub use facts::TurnStatus;
|
||||
pub use facts::TurnSteerRejectionReason;
|
||||
pub use facts::TurnSteerResult;
|
||||
pub use facts::TurnTokenUsageFact;
|
||||
pub use facts::build_track_events_context;
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -7,6 +7,7 @@ use crate::events::CodexPluginUsedEventRequest;
|
||||
use crate::events::CodexRuntimeMetadata;
|
||||
use crate::events::CodexTurnEventParams;
|
||||
use crate::events::CodexTurnEventRequest;
|
||||
use crate::events::CodexTurnSteerEventRequest;
|
||||
use crate::events::SkillInvocationEventParams;
|
||||
use crate::events::SkillInvocationEventRequest;
|
||||
use crate::events::ThreadInitializationMode;
|
||||
@@ -16,26 +17,34 @@ use crate::events::TrackEventRequest;
|
||||
use crate::events::codex_app_metadata;
|
||||
use crate::events::codex_plugin_metadata;
|
||||
use crate::events::codex_plugin_used_metadata;
|
||||
use crate::events::codex_turn_steer_event_params;
|
||||
use crate::events::plugin_state_event_type;
|
||||
use crate::events::subagent_thread_started_event_request;
|
||||
use crate::events::thread_source_name;
|
||||
use crate::facts::AnalyticsFact;
|
||||
use crate::facts::AppMentionedInput;
|
||||
use crate::facts::AppUsedInput;
|
||||
use crate::facts::CodexTurnSteerEvent;
|
||||
use crate::facts::CustomAnalyticsFact;
|
||||
use crate::facts::PluginState;
|
||||
use crate::facts::PluginStateChangedInput;
|
||||
use crate::facts::PluginUsedInput;
|
||||
use crate::facts::SkillInvokedInput;
|
||||
use crate::facts::SubAgentThreadStartedInput;
|
||||
use crate::facts::TrackEventsContext;
|
||||
use crate::facts::TurnResolvedConfigFact;
|
||||
use crate::facts::TurnStatus;
|
||||
use crate::facts::TurnSteerRejectionReason;
|
||||
use crate::facts::TurnSteerResult;
|
||||
use crate::facts::TurnTokenUsageFact;
|
||||
use codex_app_server_protocol::ClientRequest;
|
||||
use codex_app_server_protocol::ClientResponse;
|
||||
use codex_app_server_protocol::CodexErrorInfo;
|
||||
use codex_app_server_protocol::InitializeParams;
|
||||
use codex_app_server_protocol::JSONRPCErrorError;
|
||||
use codex_app_server_protocol::RequestId;
|
||||
use codex_app_server_protocol::ServerNotification;
|
||||
use codex_app_server_protocol::TurnSteerResponse;
|
||||
use codex_app_server_protocol::UserInput;
|
||||
use codex_git_utils::collect_git_info;
|
||||
use codex_git_utils::get_git_repo_root;
|
||||
@@ -46,9 +55,12 @@ use codex_protocol::config_types::ReasoningSummary;
|
||||
use codex_protocol::protocol::SandboxPolicy;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use codex_protocol::protocol::SkillScope;
|
||||
use codex_protocol::protocol::TokenUsage;
|
||||
use sha1::Digest;
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
use std::time::SystemTime;
|
||||
use std::time::UNIX_EPOCH;
|
||||
|
||||
#[derive(Default)]
|
||||
pub(crate) struct AnalyticsReducer {
|
||||
@@ -64,6 +76,7 @@ struct ConnectionState {
|
||||
|
||||
enum RequestState {
|
||||
TurnStart(PendingTurnStartState),
|
||||
TurnSteer(PendingTurnSteerState),
|
||||
}
|
||||
|
||||
struct PendingTurnStartState {
|
||||
@@ -71,6 +84,13 @@ struct PendingTurnStartState {
|
||||
num_input_images: usize,
|
||||
}
|
||||
|
||||
struct PendingTurnSteerState {
|
||||
thread_id: String,
|
||||
expected_turn_id: String,
|
||||
num_input_images: usize,
|
||||
created_at: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct CompletedTurnState {
|
||||
status: Option<TurnStatus>,
|
||||
@@ -85,7 +105,9 @@ struct TurnState {
|
||||
num_input_images: Option<usize>,
|
||||
resolved_config: Option<TurnResolvedConfigFact>,
|
||||
started_at: Option<u64>,
|
||||
token_usage: Option<TokenUsage>,
|
||||
completed: Option<CompletedTurnState>,
|
||||
steer_count: usize,
|
||||
}
|
||||
|
||||
impl AnalyticsReducer {
|
||||
@@ -119,6 +141,13 @@ impl AnalyticsReducer {
|
||||
} => {
|
||||
self.ingest_response(connection_id, *response, out);
|
||||
}
|
||||
AnalyticsFact::ErrorResponse {
|
||||
connection_id,
|
||||
request_id,
|
||||
error,
|
||||
} => {
|
||||
self.ingest_error_response(connection_id, request_id, error, out);
|
||||
}
|
||||
AnalyticsFact::Notification(notification) => {
|
||||
self.ingest_notification(*notification, out);
|
||||
}
|
||||
@@ -129,6 +158,9 @@ impl AnalyticsReducer {
|
||||
CustomAnalyticsFact::TurnResolvedConfig(input) => {
|
||||
self.ingest_turn_resolved_config(*input, out);
|
||||
}
|
||||
CustomAnalyticsFact::TurnTokenUsage(input) => {
|
||||
self.ingest_turn_token_usage(*input, out);
|
||||
}
|
||||
CustomAnalyticsFact::SkillInvoked(input) => {
|
||||
self.ingest_skill_invoked(input, out).await;
|
||||
}
|
||||
@@ -189,22 +221,29 @@ impl AnalyticsReducer {
|
||||
request_id: RequestId,
|
||||
request: ClientRequest,
|
||||
) {
|
||||
let ClientRequest::TurnStart { params, .. } = request else {
|
||||
return;
|
||||
};
|
||||
self.requests.insert(
|
||||
(connection_id, request_id),
|
||||
RequestState::TurnStart(PendingTurnStartState {
|
||||
thread_id: params.thread_id,
|
||||
num_input_images: params
|
||||
.input
|
||||
.iter()
|
||||
.filter(|item| {
|
||||
matches!(item, UserInput::Image { .. } | UserInput::LocalImage { .. })
|
||||
})
|
||||
.count(),
|
||||
}),
|
||||
);
|
||||
match request {
|
||||
ClientRequest::TurnStart { params, .. } => {
|
||||
self.requests.insert(
|
||||
(connection_id, request_id),
|
||||
RequestState::TurnStart(PendingTurnStartState {
|
||||
thread_id: params.thread_id,
|
||||
num_input_images: num_input_images(¶ms.input),
|
||||
}),
|
||||
);
|
||||
}
|
||||
ClientRequest::TurnSteer { params, .. } => {
|
||||
self.requests.insert(
|
||||
(connection_id, request_id),
|
||||
RequestState::TurnSteer(PendingTurnSteerState {
|
||||
thread_id: params.thread_id,
|
||||
expected_turn_id: params.expected_turn_id,
|
||||
num_input_images: num_input_images(¶ms.input),
|
||||
created_at: now_unix_seconds(),
|
||||
}),
|
||||
);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn ingest_turn_resolved_config(
|
||||
@@ -221,7 +260,9 @@ impl AnalyticsReducer {
|
||||
num_input_images: None,
|
||||
resolved_config: None,
|
||||
started_at: None,
|
||||
token_usage: None,
|
||||
completed: None,
|
||||
steer_count: 0,
|
||||
});
|
||||
turn_state.thread_id = Some(thread_id);
|
||||
turn_state.num_input_images = Some(num_input_images);
|
||||
@@ -229,6 +270,27 @@ impl AnalyticsReducer {
|
||||
self.maybe_emit_turn_event(&turn_id, out);
|
||||
}
|
||||
|
||||
fn ingest_turn_token_usage(
|
||||
&mut self,
|
||||
input: TurnTokenUsageFact,
|
||||
out: &mut Vec<TrackEventRequest>,
|
||||
) {
|
||||
let turn_id = input.turn_id.clone();
|
||||
let turn_state = self.turns.entry(turn_id.clone()).or_insert(TurnState {
|
||||
connection_id: None,
|
||||
thread_id: None,
|
||||
num_input_images: None,
|
||||
resolved_config: None,
|
||||
started_at: None,
|
||||
token_usage: None,
|
||||
completed: None,
|
||||
steer_count: 0,
|
||||
});
|
||||
turn_state.thread_id = Some(input.thread_id);
|
||||
turn_state.token_usage = Some(input.token_usage);
|
||||
self.maybe_emit_turn_event(&turn_id, out);
|
||||
}
|
||||
|
||||
async fn ingest_skill_invoked(
|
||||
&mut self,
|
||||
input: SkillInvokedInput,
|
||||
@@ -373,17 +435,47 @@ impl AnalyticsReducer {
|
||||
num_input_images: None,
|
||||
resolved_config: None,
|
||||
started_at: None,
|
||||
token_usage: None,
|
||||
completed: None,
|
||||
steer_count: 0,
|
||||
});
|
||||
turn_state.connection_id = Some(connection_id);
|
||||
turn_state.thread_id = Some(pending_request.thread_id);
|
||||
turn_state.num_input_images = Some(pending_request.num_input_images);
|
||||
self.maybe_emit_turn_event(&turn_id, out);
|
||||
}
|
||||
ClientResponse::TurnSteer {
|
||||
request_id,
|
||||
response,
|
||||
} => {
|
||||
self.ingest_turn_steer_response(connection_id, request_id, response, out);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn ingest_error_response(
|
||||
&mut self,
|
||||
connection_id: u64,
|
||||
request_id: RequestId,
|
||||
error: JSONRPCErrorError,
|
||||
out: &mut Vec<TrackEventRequest>,
|
||||
) {
|
||||
let Some(RequestState::TurnSteer(pending_request)) =
|
||||
self.requests.remove(&(connection_id, request_id))
|
||||
else {
|
||||
return;
|
||||
};
|
||||
self.emit_turn_steer_event(
|
||||
connection_id,
|
||||
pending_request,
|
||||
/*accepted_turn_id*/ None,
|
||||
TurnSteerResult::Rejected,
|
||||
rejection_reason_from_error(&error),
|
||||
out,
|
||||
);
|
||||
}
|
||||
|
||||
fn ingest_notification(
|
||||
&mut self,
|
||||
notification: ServerNotification,
|
||||
@@ -397,7 +489,9 @@ impl AnalyticsReducer {
|
||||
num_input_images: None,
|
||||
resolved_config: None,
|
||||
started_at: None,
|
||||
token_usage: None,
|
||||
completed: None,
|
||||
steer_count: 0,
|
||||
});
|
||||
turn_state.started_at = notification
|
||||
.turn
|
||||
@@ -414,7 +508,9 @@ impl AnalyticsReducer {
|
||||
num_input_images: None,
|
||||
resolved_config: None,
|
||||
started_at: None,
|
||||
token_usage: None,
|
||||
completed: None,
|
||||
steer_count: 0,
|
||||
});
|
||||
turn_state.completed = Some(CompletedTurnState {
|
||||
status: analytics_turn_status(notification.turn.status),
|
||||
@@ -470,6 +566,70 @@ impl AnalyticsReducer {
|
||||
));
|
||||
}
|
||||
|
||||
fn ingest_turn_steer_response(
|
||||
&mut self,
|
||||
connection_id: u64,
|
||||
request_id: RequestId,
|
||||
response: TurnSteerResponse,
|
||||
out: &mut Vec<TrackEventRequest>,
|
||||
) {
|
||||
let Some(RequestState::TurnSteer(pending_request)) =
|
||||
self.requests.remove(&(connection_id, request_id))
|
||||
else {
|
||||
return;
|
||||
};
|
||||
if let Some(turn_state) = self.turns.get_mut(&response.turn_id) {
|
||||
turn_state.steer_count += 1;
|
||||
}
|
||||
self.emit_turn_steer_event(
|
||||
connection_id,
|
||||
pending_request,
|
||||
Some(response.turn_id),
|
||||
TurnSteerResult::Accepted,
|
||||
/*rejection_reason*/ None,
|
||||
out,
|
||||
);
|
||||
}
|
||||
|
||||
fn emit_turn_steer_event(
|
||||
&mut self,
|
||||
connection_id: u64,
|
||||
pending_request: PendingTurnSteerState,
|
||||
accepted_turn_id: Option<String>,
|
||||
result: TurnSteerResult,
|
||||
rejection_reason: Option<TurnSteerRejectionReason>,
|
||||
out: &mut Vec<TrackEventRequest>,
|
||||
) {
|
||||
let Some(connection_state) = self.connections.get(&connection_id) else {
|
||||
return;
|
||||
};
|
||||
let tracking = TrackEventsContext {
|
||||
model_slug: String::new(),
|
||||
thread_id: pending_request.thread_id,
|
||||
turn_id: accepted_turn_id
|
||||
.as_deref()
|
||||
.unwrap_or(pending_request.expected_turn_id.as_str())
|
||||
.to_string(),
|
||||
};
|
||||
let turn_steer = CodexTurnSteerEvent {
|
||||
expected_turn_id: Some(pending_request.expected_turn_id),
|
||||
accepted_turn_id,
|
||||
num_input_images: pending_request.num_input_images,
|
||||
result,
|
||||
rejection_reason,
|
||||
created_at: pending_request.created_at,
|
||||
};
|
||||
out.push(TrackEventRequest::TurnSteer(CodexTurnSteerEventRequest {
|
||||
event_type: "codex_turn_steer_event",
|
||||
event_params: codex_turn_steer_event_params(
|
||||
connection_state.app_server_client.clone(),
|
||||
connection_state.runtime.clone(),
|
||||
&tracking,
|
||||
turn_steer,
|
||||
),
|
||||
}));
|
||||
}
|
||||
|
||||
fn maybe_emit_turn_event(&mut self, turn_id: &str, out: &mut Vec<TrackEventRequest>) {
|
||||
let Some(turn_state) = self.turns.get(turn_id) else {
|
||||
return;
|
||||
@@ -532,6 +692,7 @@ fn codex_turn_event_params(
|
||||
personality,
|
||||
is_first_turn,
|
||||
} = resolved_config;
|
||||
let token_usage = turn_state.token_usage.clone();
|
||||
CodexTurnEventParams {
|
||||
thread_id,
|
||||
turn_id,
|
||||
@@ -554,7 +715,7 @@ fn codex_turn_event_params(
|
||||
is_first_turn,
|
||||
status: completed.status,
|
||||
turn_error: completed.turn_error,
|
||||
steer_count: None,
|
||||
steer_count: Some(turn_state.steer_count),
|
||||
total_tool_call_count: None,
|
||||
shell_command_count: None,
|
||||
file_change_count: None,
|
||||
@@ -563,6 +724,21 @@ fn codex_turn_event_params(
|
||||
subagent_tool_call_count: None,
|
||||
web_search_count: None,
|
||||
image_generation_count: None,
|
||||
input_tokens: token_usage
|
||||
.as_ref()
|
||||
.map(|token_usage| token_usage.input_tokens),
|
||||
cached_input_tokens: token_usage
|
||||
.as_ref()
|
||||
.map(|token_usage| token_usage.cached_input_tokens),
|
||||
output_tokens: token_usage
|
||||
.as_ref()
|
||||
.map(|token_usage| token_usage.output_tokens),
|
||||
reasoning_output_tokens: token_usage
|
||||
.as_ref()
|
||||
.map(|token_usage| token_usage.reasoning_output_tokens),
|
||||
total_tokens: token_usage
|
||||
.as_ref()
|
||||
.map(|token_usage| token_usage.total_tokens),
|
||||
duration_ms: completed.duration_ms,
|
||||
started_at,
|
||||
completed_at: Some(completed.completed_at),
|
||||
@@ -608,6 +784,52 @@ fn analytics_turn_status(status: codex_app_server_protocol::TurnStatus) -> Optio
|
||||
}
|
||||
}
|
||||
|
||||
fn num_input_images(input: &[UserInput]) -> usize {
|
||||
input
|
||||
.iter()
|
||||
.filter(|item| matches!(item, UserInput::Image { .. } | UserInput::LocalImage { .. }))
|
||||
.count()
|
||||
}
|
||||
|
||||
fn now_unix_seconds() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
}
|
||||
|
||||
fn rejection_reason_from_error(error: &JSONRPCErrorError) -> Option<TurnSteerRejectionReason> {
|
||||
if error.message == "no active turn to steer" {
|
||||
return Some(TurnSteerRejectionReason::NoActiveTurn);
|
||||
}
|
||||
if error.message.starts_with("expected active turn id ") {
|
||||
return Some(TurnSteerRejectionReason::ExpectedTurnMismatch);
|
||||
}
|
||||
if error.message == "input must not be empty" {
|
||||
return Some(TurnSteerRejectionReason::EmptyInput);
|
||||
}
|
||||
error.data.as_ref().and_then(|data| {
|
||||
if data
|
||||
.get("input_error_code")
|
||||
.and_then(|input_error_code| input_error_code.as_str())
|
||||
== Some("input_too_large")
|
||||
{
|
||||
return Some(TurnSteerRejectionReason::InputTooLarge);
|
||||
}
|
||||
|
||||
let turn_kind = data
|
||||
.get("codexErrorInfo")?
|
||||
.get("activeTurnNotSteerable")?
|
||||
.get("turnKind")?
|
||||
.as_str()?;
|
||||
match turn_kind {
|
||||
"review" => Some(TurnSteerRejectionReason::NonSteerableReview),
|
||||
"compact" => Some(TurnSteerRejectionReason::NonSteerableCompact),
|
||||
_ => None,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn skill_id_for_local_skill(
|
||||
repo_url: Option<&str>,
|
||||
repo_root: Option<&Path>,
|
||||
|
||||
@@ -35,7 +35,7 @@ use codex_app_server_protocol::CancelLoginAccountResponse;
|
||||
use codex_app_server_protocol::CancelLoginAccountStatus;
|
||||
use codex_app_server_protocol::ClientRequest;
|
||||
use codex_app_server_protocol::ClientResponse;
|
||||
use codex_app_server_protocol::CodexErrorInfo as AppServerCodexErrorInfo;
|
||||
use codex_app_server_protocol::CodexErrorInfo;
|
||||
use codex_app_server_protocol::CollaborationModeListParams;
|
||||
use codex_app_server_protocol::CollaborationModeListResponse;
|
||||
use codex_app_server_protocol::CommandExecParams;
|
||||
@@ -497,6 +497,16 @@ impl CodexMessageProcessor {
|
||||
}
|
||||
}
|
||||
|
||||
fn track_error_response(&self, request_id: &ConnectionRequestId, error: &JSONRPCErrorError) {
|
||||
if self.config.features.enabled(Feature::GeneralAnalytics) {
|
||||
self.analytics_events_client.track_error_response(
|
||||
request_id.connection_id.0,
|
||||
request_id.request_id.clone(),
|
||||
error.clone(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn load_thread(
|
||||
&self,
|
||||
thread_id: &str,
|
||||
@@ -6706,23 +6716,27 @@ impl CodexMessageProcessor {
|
||||
let (_, thread) = match self.load_thread(¶ms.thread_id).await {
|
||||
Ok(v) => v,
|
||||
Err(error) => {
|
||||
self.track_error_response(&request_id, &error);
|
||||
self.outgoing.send_error(request_id, error).await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
if params.expected_turn_id.is_empty() {
|
||||
self.send_invalid_request_error(
|
||||
request_id,
|
||||
"expectedTurnId must not be empty".to_string(),
|
||||
)
|
||||
.await;
|
||||
let error = JSONRPCErrorError {
|
||||
code: INVALID_REQUEST_ERROR_CODE,
|
||||
message: "expectedTurnId must not be empty".to_string(),
|
||||
data: None,
|
||||
};
|
||||
self.track_error_response(&request_id, &error);
|
||||
self.outgoing.send_error(request_id, error).await;
|
||||
return;
|
||||
}
|
||||
self.outgoing
|
||||
.record_request_turn_id(&request_id, ¶ms.expected_turn_id)
|
||||
.await;
|
||||
if let Err(error) = Self::validate_v2_input_limit(¶ms.input) {
|
||||
self.track_error_response(&request_id, &error);
|
||||
self.outgoing.send_error(request_id, error).await;
|
||||
return;
|
||||
}
|
||||
@@ -6739,6 +6753,15 @@ impl CodexMessageProcessor {
|
||||
{
|
||||
Ok(turn_id) => {
|
||||
let response = TurnSteerResponse { turn_id };
|
||||
if self.config.features.enabled(Feature::GeneralAnalytics) {
|
||||
self.analytics_events_client.track_response(
|
||||
request_id.connection_id.0,
|
||||
ClientResponse::TurnSteer {
|
||||
request_id: request_id.request_id.clone(),
|
||||
response: response.clone(),
|
||||
},
|
||||
);
|
||||
}
|
||||
self.outgoing.send_response(request_id, response).await;
|
||||
}
|
||||
Err(err) => {
|
||||
@@ -6764,11 +6787,9 @@ impl CodexMessageProcessor {
|
||||
};
|
||||
let error = TurnError {
|
||||
message: message.clone(),
|
||||
codex_error_info: Some(
|
||||
AppServerCodexErrorInfo::ActiveTurnNotSteerable {
|
||||
turn_kind: turn_kind.into(),
|
||||
},
|
||||
),
|
||||
codex_error_info: Some(CodexErrorInfo::ActiveTurnNotSteerable {
|
||||
turn_kind: turn_kind.into(),
|
||||
}),
|
||||
additional_details: None,
|
||||
};
|
||||
let data = match serde_json::to_value(error) {
|
||||
@@ -6794,6 +6815,7 @@ impl CodexMessageProcessor {
|
||||
message,
|
||||
data,
|
||||
};
|
||||
self.track_error_response(&request_id, &error);
|
||||
self.outgoing.send_error(request_id, error).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -678,7 +678,8 @@ impl MessageProcessor {
|
||||
return;
|
||||
}
|
||||
if self.config.features.enabled(Feature::GeneralAnalytics)
|
||||
&& let ClientRequest::TurnStart { request_id, .. } = &codex_request
|
||||
&& let ClientRequest::TurnStart { request_id, .. }
|
||||
| ClientRequest::TurnSteer { request_id, .. } = &codex_request
|
||||
{
|
||||
self.analytics_events_client.track_request(
|
||||
connection_id.0,
|
||||
|
||||
@@ -310,6 +310,11 @@ async fn turn_start_tracks_turn_event_analytics() -> Result<()> {
|
||||
assert!(event["event_params"]["started_at"].as_u64().is_some());
|
||||
assert!(event["event_params"]["completed_at"].as_u64().is_some());
|
||||
assert!(event["event_params"]["duration_ms"].as_u64().is_some());
|
||||
assert_eq!(event["event_params"]["input_tokens"], 0);
|
||||
assert_eq!(event["event_params"]["cached_input_tokens"], 0);
|
||||
assert_eq!(event["event_params"]["output_tokens"], 0);
|
||||
assert_eq!(event["event_params"]["reasoning_output_tokens"], 0);
|
||||
assert_eq!(event["event_params"]["total_tokens"], 0);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ use app_test_support::create_mock_responses_server_sequence;
|
||||
use app_test_support::create_mock_responses_server_sequence_unchecked;
|
||||
use app_test_support::create_shell_command_sse_response;
|
||||
use app_test_support::to_response;
|
||||
use app_test_support::write_mock_responses_config_toml_with_chatgpt_base_url;
|
||||
use codex_app_server::INPUT_TOO_LARGE_ERROR_CODE;
|
||||
use codex_app_server::INVALID_PARAMS_ERROR_CODE;
|
||||
use codex_app_server_protocol::JSONRPCError;
|
||||
@@ -23,6 +24,9 @@ use codex_protocol::user_input::MAX_USER_INPUT_TEXT_CHARS;
|
||||
use tempfile::TempDir;
|
||||
use tokio::time::timeout;
|
||||
|
||||
use super::analytics::enable_analytics_capture;
|
||||
use super::analytics::wait_for_analytics_event;
|
||||
|
||||
const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
|
||||
|
||||
#[tokio::test]
|
||||
@@ -32,7 +36,12 @@ async fn turn_steer_requires_active_turn() -> Result<()> {
|
||||
std::fs::create_dir(&codex_home)?;
|
||||
|
||||
let server = create_mock_responses_server_sequence(vec![]).await;
|
||||
create_config_toml(&codex_home, &server.uri())?;
|
||||
write_mock_responses_config_toml_with_chatgpt_base_url(
|
||||
&codex_home,
|
||||
&server.uri(),
|
||||
&server.uri(),
|
||||
)?;
|
||||
enable_analytics_capture(&server, &codex_home).await?;
|
||||
|
||||
let mut mcp = McpProcess::new(&codex_home).await?;
|
||||
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
|
||||
@@ -52,7 +61,7 @@ async fn turn_steer_requires_active_turn() -> Result<()> {
|
||||
|
||||
let steer_req = mcp
|
||||
.send_turn_steer_request(TurnSteerParams {
|
||||
thread_id: thread.id,
|
||||
thread_id: thread.id.clone(),
|
||||
input: vec![V2UserInput::Text {
|
||||
text: "steer".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
@@ -67,6 +76,21 @@ async fn turn_steer_requires_active_turn() -> Result<()> {
|
||||
.await??;
|
||||
assert_eq!(steer_err.error.code, -32600);
|
||||
|
||||
let event =
|
||||
wait_for_analytics_event(&server, DEFAULT_READ_TIMEOUT, "codex_turn_steer_event").await?;
|
||||
assert_eq!(event["event_params"]["thread_id"], thread.id);
|
||||
assert_eq!(event["event_params"]["result"], "rejected");
|
||||
assert_eq!(event["event_params"]["num_input_images"], 0);
|
||||
assert_eq!(
|
||||
event["event_params"]["expected_turn_id"],
|
||||
"turn-does-not-exist"
|
||||
);
|
||||
assert_eq!(
|
||||
event["event_params"]["accepted_turn_id"],
|
||||
serde_json::Value::Null
|
||||
);
|
||||
assert_eq!(event["event_params"]["rejection_reason"], "no_active_turn");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -95,7 +119,12 @@ async fn turn_steer_rejects_oversized_text_input() -> Result<()> {
|
||||
"call_sleep",
|
||||
)?])
|
||||
.await;
|
||||
create_config_toml(&codex_home, &server.uri())?;
|
||||
write_mock_responses_config_toml_with_chatgpt_base_url(
|
||||
&codex_home,
|
||||
&server.uri(),
|
||||
&server.uri(),
|
||||
)?;
|
||||
enable_analytics_capture(&server, &codex_home).await?;
|
||||
|
||||
let mut mcp = McpProcess::new(&codex_home).await?;
|
||||
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
|
||||
@@ -198,7 +227,12 @@ async fn turn_steer_returns_active_turn_id() -> Result<()> {
|
||||
"call_sleep",
|
||||
)?])
|
||||
.await;
|
||||
create_config_toml(&codex_home, &server.uri())?;
|
||||
write_mock_responses_config_toml_with_chatgpt_base_url(
|
||||
&codex_home,
|
||||
&server.uri(),
|
||||
&server.uri(),
|
||||
)?;
|
||||
enable_analytics_capture(&server, &codex_home).await?;
|
||||
|
||||
let mut mcp = McpProcess::new(&codex_home).await?;
|
||||
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
|
||||
@@ -258,31 +292,20 @@ async fn turn_steer_returns_active_turn_id() -> Result<()> {
|
||||
let steer: TurnSteerResponse = to_response::<TurnSteerResponse>(steer_resp)?;
|
||||
assert_eq!(steer.turn_id, turn.id);
|
||||
|
||||
let event =
|
||||
wait_for_analytics_event(&server, DEFAULT_READ_TIMEOUT, "codex_turn_steer_event").await?;
|
||||
assert_eq!(event["event_params"]["thread_id"], thread.id);
|
||||
assert_eq!(event["event_params"]["result"], "accepted");
|
||||
assert_eq!(event["event_params"]["num_input_images"], 0);
|
||||
assert_eq!(event["event_params"]["expected_turn_id"], turn.id);
|
||||
assert_eq!(event["event_params"]["accepted_turn_id"], turn.id);
|
||||
assert_eq!(
|
||||
event["event_params"]["rejection_reason"],
|
||||
serde_json::Value::Null
|
||||
);
|
||||
|
||||
mcp.interrupt_turn_and_wait_for_aborted(thread.id, steer.turn_id, DEFAULT_READ_TIMEOUT)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn create_config_toml(codex_home: &std::path::Path, server_uri: &str) -> std::io::Result<()> {
|
||||
let config_toml = codex_home.join("config.toml");
|
||||
std::fs::write(
|
||||
config_toml,
|
||||
format!(
|
||||
r#"
|
||||
model = "mock-model"
|
||||
approval_policy = "never"
|
||||
sandbox_mode = "danger-full-access"
|
||||
|
||||
model_provider = "mock_provider"
|
||||
|
||||
[model_providers.mock_provider]
|
||||
name = "Mock provider for test"
|
||||
base_url = "{server_uri}/v1"
|
||||
wire_api = "responses"
|
||||
request_max_retries = 0
|
||||
stream_max_retries = 0
|
||||
"#
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -30,6 +30,7 @@ use crate::hook_runtime::record_pending_input;
|
||||
use crate::state::ActiveTurn;
|
||||
use crate::state::RunningTask;
|
||||
use crate::state::TaskKind;
|
||||
use codex_analytics::TurnTokenUsageFact;
|
||||
use codex_login::AuthManager;
|
||||
use codex_models_manager::manager::ModelsManager;
|
||||
use codex_otel::SessionTelemetry;
|
||||
@@ -485,6 +486,13 @@ impl Session {
|
||||
- token_usage_at_turn_start.total_tokens)
|
||||
.max(0),
|
||||
};
|
||||
self.services
|
||||
.analytics_events_client
|
||||
.track_turn_token_usage(TurnTokenUsageFact {
|
||||
turn_id: turn_context.sub_id.clone(),
|
||||
thread_id: self.conversation_id.to_string(),
|
||||
token_usage: turn_token_usage.clone(),
|
||||
});
|
||||
self.services.session_telemetry.histogram(
|
||||
TURN_TOKEN_USAGE_METRIC,
|
||||
turn_token_usage.total_tokens,
|
||||
|
||||
Reference in New Issue
Block a user