mirrors/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-05-02 12:21:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

[codex-analytics] add steering metadata

Browse Source
This commit is contained in:
rhan-oai
2026-04-06 12:26:33 -07:00
parent 59cfda78e2
commit 9e7e31966f
8 changed files with 525 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

193
codex-rs/analytics/src/analytics_client_tests.rs
View File

@@ -7,6 +7,7 @@ use crate::events::CodexPluginEventRequest;
use crate::events::CodexPluginUsedEventRequest;
use crate::events::CodexRuntimeMetadata;
use crate::events::CodexTurnEventRequest;
use crate::events::CodexTurnSteerEventRequest;
use crate::events::ThreadInitializationMode;
use crate::events::ThreadInitializedEvent;
use crate::events::ThreadInitializedEventParams;
@@ -14,11 +15,13 @@ use crate::events::TrackEventRequest;
use crate::events::codex_app_metadata;
use crate::events::codex_plugin_metadata;
use crate::events::codex_plugin_used_metadata;
use crate::events::codex_turn_steer_event_params;
use crate::events::subagent_thread_started_event_request;
use crate::facts::AnalyticsFact;
use crate::facts::AppInvocation;
use crate::facts::AppMentionedInput;
use crate::facts::AppUsedInput;
use crate::facts::CodexTurnSteerEvent;
use crate::facts::CustomAnalyticsFact;
use crate::facts::InvocationType;
use crate::facts::PluginState;
@@ -30,6 +33,9 @@ use crate::facts::SubAgentThreadStartedInput;
use crate::facts::TrackEventsContext;
use crate::facts::TurnResolvedConfigFact;
use crate::facts::TurnStatus;
use crate::facts::TurnSteerInput;
use crate::facts::TurnSteerRejectionReason;
use crate::facts::TurnSteerResult;
use crate::facts::TurnSubmissionType;
use crate::reducer::AnalyticsReducer;
use crate::reducer::normalize_path_for_skill_id;
@@ -1079,7 +1085,7 @@ fn turn_event_serializes_expected_shape() {
is_first_turn: true,
status: Some(TurnStatus::Completed),
turn_error: None,
steer_count: None,
steer_count: Some(0),
total_tool_call_count: None,
shell_command_count: None,
file_change_count: None,
@@ -1125,7 +1131,7 @@ fn turn_event_serializes_expected_shape() {
"is_first_turn": true,
"status": "completed",
"turn_error": null,
"steer_count": null,
"steer_count": 0,
"total_tool_call_count": null,
"shell_command_count": null,
"file_change_count": null,
@@ -1147,6 +1153,90 @@ fn turn_event_serializes_expected_shape() {
);
}
#[test]
fn turn_steer_event_serializes_expected_shape() {
let tracking = TrackEventsContext {
model_slug: "gpt-5".to_string(),
thread_id: "thread-2".to_string(),
turn_id: "turn-2".to_string(),
};
let event = TrackEventRequest::TurnSteer(CodexTurnSteerEventRequest {
event_type: "codex_turn_steer_event",
event_params: codex_turn_steer_event_params(
&tracking,
CodexTurnSteerEvent {
expected_turn_id: Some("turn-2".to_string()),
accepted_turn_id: Some("turn-2".to_string()),
num_input_images: 2,
result: TurnSteerResult::Accepted,
rejection_reason: None,
created_at: 1_716_000_123,
},
),
});
let payload = serde_json::to_value(&event).expect("serialize turn steer event");
assert_eq!(
payload,
json!({
"event_type": "codex_turn_steer_event",
"event_params": {
"thread_id": "thread-2",
"expected_turn_id": "turn-2",
"accepted_turn_id": "turn-2",
"product_client_id": originator().value,
"num_input_images": 2,
"result": "accepted",
"rejection_reason": null,
"created_at": 1_716_000_123
}
})
);
}
#[test]
fn rejected_turn_steer_event_serializes_expected_shape() {
let tracking = TrackEventsContext {
model_slug: "gpt-5".to_string(),
thread_id: "thread-3".to_string(),
turn_id: "turn-3".to_string(),
};
let event = TrackEventRequest::TurnSteer(CodexTurnSteerEventRequest {
event_type: "codex_turn_steer_event",
event_params: codex_turn_steer_event_params(
&tracking,
CodexTurnSteerEvent {
expected_turn_id: Some("turn-expected".to_string()),
accepted_turn_id: None,
num_input_images: 1,
result: TurnSteerResult::Rejected,
rejection_reason: Some(TurnSteerRejectionReason::ExpectedTurnMismatch),
created_at: 1_716_000_124,
},
),
});
let payload = serde_json::to_value(&event).expect("serialize rejected turn steer event");
assert_eq!(
payload,
json!({
"event_type": "codex_turn_steer_event",
"event_params": {
"thread_id": "thread-3",
"expected_turn_id": "turn-expected",
"accepted_turn_id": null,
"product_client_id": originator().value,
"num_input_images": 1,
"result": "rejected",
"rejection_reason": "expected_turn_mismatch",
"created_at": 1_716_000_124
}
})
);
}
#[tokio::test]
async fn turn_lifecycle_emits_turn_event() {
let mut reducer = AnalyticsReducer::default();
@@ -1184,6 +1274,7 @@ async fn turn_lifecycle_emits_turn_event() {
);
assert_eq!(payload["event_params"]["num_input_images"], json!(1));
assert_eq!(payload["event_params"]["status"], json!("completed"));
assert_eq!(payload["event_params"]["steer_count"], json!(0));
assert_eq!(payload["event_params"]["started_at"], json!(455));
assert_eq!(payload["event_params"]["completed_at"], json!(456));
assert_eq!(payload["event_params"]["duration_ms"], json!(1234));
@@ -1197,6 +1288,104 @@ async fn turn_lifecycle_emits_turn_event() {
assert_eq!(payload["event_params"]["total_tokens"], json!(321));
}
#[tokio::test]
async fn accepted_steers_increment_turn_steer_count() {
let mut reducer = AnalyticsReducer::default();
let mut out = Vec::new();
ingest_turn_prerequisites(
&mut reducer,
&mut out,
/*include_initialize*/ true,
/*include_resolved_config*/ true,
/*include_started*/ true,
/*include_token_usage*/ false,
)
.await;
reducer
.ingest(
AnalyticsFact::Custom(CustomAnalyticsFact::TurnSteer(TurnSteerInput {
tracking: TrackEventsContext {
model_slug: "gpt-5".to_string(),
thread_id: "thread-2".to_string(),
turn_id: "turn-2".to_string(),
},
turn_steer: CodexTurnSteerEvent {
expected_turn_id: Some("turn-2".to_string()),
accepted_turn_id: Some("turn-2".to_string()),
num_input_images: 0,
result: TurnSteerResult::Accepted,
rejection_reason: None,
created_at: 1,
},
})),
&mut out,
)
.await;
reducer
.ingest(
AnalyticsFact::Custom(CustomAnalyticsFact::TurnSteer(TurnSteerInput {
tracking: TrackEventsContext {
model_slug: "gpt-5".to_string(),
thread_id: "thread-2".to_string(),
turn_id: "turn-2".to_string(),
},
turn_steer: CodexTurnSteerEvent {
expected_turn_id: None,
accepted_turn_id: None,
num_input_images: 0,
result: TurnSteerResult::Rejected,
rejection_reason: Some(TurnSteerRejectionReason::NoActiveTurn),
created_at: 2,
},
})),
&mut out,
)
.await;
reducer
.ingest(
AnalyticsFact::Custom(CustomAnalyticsFact::TurnSteer(TurnSteerInput {
tracking: TrackEventsContext {
model_slug: "gpt-5".to_string(),
thread_id: "thread-2".to_string(),
turn_id: "turn-2".to_string(),
},
turn_steer: CodexTurnSteerEvent {
expected_turn_id: Some("turn-2".to_string()),
accepted_turn_id: Some("turn-2".to_string()),
num_input_images: 1,
result: TurnSteerResult::Accepted,
rejection_reason: None,
created_at: 3,
},
})),
&mut out,
)
.await;
reducer
.ingest(
AnalyticsFact::Notification(Box::new(sample_turn_completed_notification(
"thread-2",
"turn-2",
AppServerTurnStatus::Completed,
/*codex_error_info*/ None,
))),
&mut out,
)
.await;
let turn_event = out
.iter()
.find(|event| matches!(event, TrackEventRequest::TurnEvent(_)))
.expect("turn event should be emitted");
let payload = serde_json::to_value(turn_event).expect("serialize turn event");
assert_eq!(payload["event_params"]["steer_count"], json!(2));
}
#[tokio::test]
async fn queued_submission_type_emits_queued_turn_event() {
let mut reducer = AnalyticsReducer::default();

11
codex-rs/analytics/src/client.rs
View File

@@ -6,6 +6,7 @@ use crate::facts::AnalyticsFact;
use crate::facts::AppInvocation;
use crate::facts::AppMentionedInput;
use crate::facts::AppUsedInput;
use crate::facts::CodexTurnSteerEvent;
use crate::facts::CustomAnalyticsFact;
use crate::facts::PluginState;
use crate::facts::PluginStateChangedInput;
@@ -14,6 +15,7 @@ use crate::facts::SkillInvokedInput;
use crate::facts::SubAgentThreadStartedInput;
use crate::facts::TrackEventsContext;
use crate::facts::TurnResolvedConfigFact;
use crate::facts::TurnSteerInput;
use crate::reducer::AnalyticsReducer;
use codex_app_server_protocol::ClientRequest;
use codex_app_server_protocol::ClientResponse;
@@ -196,6 +198,15 @@ impl AnalyticsEventsClient {
));
}
pub fn track_turn_steer(&self, tracking: TrackEventsContext, turn_steer: CodexTurnSteerEvent) {
self.record_fact(AnalyticsFact::Custom(CustomAnalyticsFact::TurnSteer(
TurnSteerInput {
tracking,
turn_steer,
},
)));
}
pub fn track_plugin_installed(&self, plugin: PluginTelemetryMetadata) {
self.record_fact(AnalyticsFact::Custom(
CustomAnalyticsFact::PluginStateChanged(PluginStateChangedInput {

38
codex-rs/analytics/src/events.rs
View File

@@ -1,9 +1,12 @@
use crate::facts::AppInvocation;
use crate::facts::CodexTurnSteerEvent;
use crate::facts::InvocationType;
use crate::facts::PluginState;
use crate::facts::SubAgentThreadStartedInput;
use crate::facts::TrackEventsContext;
use crate::facts::TurnStatus;
use crate::facts::TurnSteerRejectionReason;
use crate::facts::TurnSteerResult;
use crate::facts::TurnSubmissionType;
use codex_app_server_protocol::CodexErrorInfo;
use codex_login::default_client::originator;
@@ -41,6 +44,7 @@ pub(crate) enum TrackEventRequest {
AppMentioned(CodexAppMentionedEventRequest),
AppUsed(CodexAppUsedEventRequest),
TurnEvent(Box<CodexTurnEventRequest>),
TurnSteer(CodexTurnSteerEventRequest),
PluginUsed(CodexPluginUsedEventRequest),
PluginInstalled(CodexPluginEventRequest),
PluginUninstalled(CodexPluginEventRequest),
@@ -172,6 +176,24 @@ pub(crate) struct CodexTurnEventRequest {
pub(crate) event_params: CodexTurnEventParams,
}
#[derive(Serialize)]
pub(crate) struct CodexTurnSteerEventParams {
pub(crate) thread_id: String,
pub(crate) expected_turn_id: Option<String>,
pub(crate) accepted_turn_id: Option<String>,
pub(crate) product_client_id: String,
pub(crate) num_input_images: usize,
pub(crate) result: TurnSteerResult,
pub(crate) rejection_reason: Option<TurnSteerRejectionReason>,
pub(crate) created_at: u64,
}
#[derive(Serialize)]
pub(crate) struct CodexTurnSteerEventRequest {
pub(crate) event_type: &'static str,
pub(crate) event_params: CodexTurnSteerEventParams,
}
#[derive(Serialize)]
pub(crate) struct CodexPluginMetadata {
pub(crate) plugin_id: Option<String>,
@@ -263,6 +285,22 @@ pub(crate) fn codex_plugin_used_metadata(
}
}
pub(crate) fn codex_turn_steer_event_params(
tracking: &TrackEventsContext,
turn_steer: CodexTurnSteerEvent,
) -> CodexTurnSteerEventParams {
CodexTurnSteerEventParams {
thread_id: tracking.thread_id.clone(),
expected_turn_id: turn_steer.expected_turn_id,
accepted_turn_id: turn_steer.accepted_turn_id,
product_client_id: originator().value,
num_input_images: turn_steer.num_input_images,
result: turn_steer.result,
rejection_reason: turn_steer.rejection_reason,
created_at: turn_steer.created_at,
}
}
pub(crate) fn thread_source_name(thread_source: &SessionSource) -> Option<&'static str> {
match thread_source {
SessionSource::Cli | SessionSource::VSCode | SessionSource::Exec => Some("user"),

33
codex-rs/analytics/src/facts.rs
View File

@@ -73,6 +73,33 @@ pub enum TurnStatus {
Interrupted,
}
#[derive(Clone, Copy, Debug, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum TurnSteerResult {
Accepted,
Rejected,
}
#[derive(Clone, Copy, Debug, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum TurnSteerRejectionReason {
NoActiveTurn,
ExpectedTurnMismatch,
NonSteerableReview,
NonSteerableCompact,
EmptyInput,
}
#[derive(Clone)]
pub struct CodexTurnSteerEvent {
pub expected_turn_id: Option<String>,
pub accepted_turn_id: Option<String>,
pub num_input_images: usize,
pub result: TurnSteerResult,
pub rejection_reason: Option<TurnSteerRejectionReason>,
pub created_at: u64,
}
#[derive(Clone, Debug)]
pub struct SkillInvocation {
pub skill_name: String,
@@ -133,6 +160,7 @@ pub(crate) enum AnalyticsFact {
pub(crate) enum CustomAnalyticsFact {
SubAgentThreadStarted(SubAgentThreadStartedInput),
TurnResolvedConfig(Box<TurnResolvedConfigFact>),
TurnSteer(TurnSteerInput),
SkillInvoked(SkillInvokedInput),
AppMentioned(AppMentionedInput),
AppUsed(AppUsedInput),
@@ -140,6 +168,11 @@ pub(crate) enum CustomAnalyticsFact {
PluginStateChanged(PluginStateChangedInput),
}
pub(crate) struct TurnSteerInput {
pub tracking: TrackEventsContext,
pub turn_steer: CodexTurnSteerEvent,
}
pub(crate) struct SkillInvokedInput {
pub tracking: TrackEventsContext,
pub invocations: Vec<SkillInvocation>,

3
codex-rs/analytics/src/lib.rs
View File

@@ -6,12 +6,15 @@ mod reducer;
pub use client::AnalyticsEventsClient;
pub use events::AppServerRpcTransport;
pub use facts::AppInvocation;
pub use facts::CodexTurnSteerEvent;
pub use facts::InvocationType;
pub use facts::SkillInvocation;
pub use facts::SubAgentThreadStartedInput;
pub use facts::TrackEventsContext;
pub use facts::TurnResolvedConfigFact;
pub use facts::TurnStatus;
pub use facts::TurnSteerRejectionReason;
pub use facts::TurnSteerResult;
pub use facts::TurnSubmissionType;
pub use facts::build_track_events_context;

32
codex-rs/analytics/src/reducer.rs
View File

@@ -7,6 +7,7 @@ use crate::events::CodexPluginUsedEventRequest;
use crate::events::CodexRuntimeMetadata;
use crate::events::CodexTurnEventParams;
use crate::events::CodexTurnEventRequest;
use crate::events::CodexTurnSteerEventRequest;
use crate::events::SkillInvocationEventParams;
use crate::events::SkillInvocationEventRequest;
use crate::events::ThreadInitializationMode;
@@ -16,6 +17,7 @@ use crate::events::TrackEventRequest;
use crate::events::codex_app_metadata;
use crate::events::codex_plugin_metadata;
use crate::events::codex_plugin_used_metadata;
use crate::events::codex_turn_steer_event_params;
use crate::events::plugin_state_event_type;
use crate::events::subagent_thread_started_event_request;
use crate::events::thread_source_name;
@@ -30,6 +32,8 @@ use crate::facts::SkillInvokedInput;
use crate::facts::SubAgentThreadStartedInput;
use crate::facts::TurnResolvedConfigFact;
use crate::facts::TurnStatus;
use crate::facts::TurnSteerInput;
use crate::facts::TurnSteerResult;
use codex_app_server_protocol::ClientRequest;
use codex_app_server_protocol::ClientResponse;
use codex_app_server_protocol::CodexErrorInfo;
@@ -88,6 +92,7 @@ struct TurnState {
started_at: Option<u64>,
token_usage: Option<TokenUsageBreakdown>,
completed: Option<CompletedTurnState>,
steer_count: usize,
}
impl AnalyticsReducer {
@@ -131,6 +136,9 @@ impl AnalyticsReducer {
CustomAnalyticsFact::TurnResolvedConfig(input) => {
self.ingest_turn_resolved_config(*input, out);
}
CustomAnalyticsFact::TurnSteer(input) => {
self.ingest_turn_steer(input, out);
}
CustomAnalyticsFact::SkillInvoked(input) => {
self.ingest_skill_invoked(input, out).await;
}
@@ -225,6 +233,7 @@ impl AnalyticsReducer {
started_at: None,
token_usage: None,
completed: None,
steer_count: 0,
});
turn_state.thread_id = Some(thread_id);
turn_state.num_input_images = Some(num_input_images);
@@ -378,6 +387,7 @@ impl AnalyticsReducer {
started_at: None,
token_usage: None,
completed: None,
steer_count: 0,
});
turn_state.connection_id = Some(connection_id);
turn_state.thread_id = Some(pending_request.thread_id);
@@ -403,6 +413,7 @@ impl AnalyticsReducer {
started_at: None,
token_usage: None,
completed: None,
steer_count: 0,
});
turn_state.started_at = notification
.turn
@@ -418,6 +429,7 @@ impl AnalyticsReducer {
started_at: None,
token_usage: None,
completed: None,
steer_count: 0,
});
turn_state.token_usage = Some(notification.token_usage.last);
}
@@ -433,6 +445,7 @@ impl AnalyticsReducer {
started_at: None,
token_usage: None,
completed: None,
steer_count: 0,
});
turn_state.completed = Some(CompletedTurnState {
status: analytics_turn_status(notification.turn.status),
@@ -488,6 +501,23 @@ impl AnalyticsReducer {
));
}
fn ingest_turn_steer(&mut self, input: TurnSteerInput, out: &mut Vec<TrackEventRequest>) {
let TurnSteerInput {
tracking,
turn_steer,
} = input;
if matches!(turn_steer.result, TurnSteerResult::Accepted)
&& let Some(accepted_turn_id) = turn_steer.accepted_turn_id.as_ref()
&& let Some(turn_state) = self.turns.get_mut(accepted_turn_id)
{
turn_state.steer_count += 1;
}
out.push(TrackEventRequest::TurnSteer(CodexTurnSteerEventRequest {
event_type: "codex_turn_steer_event",
event_params: codex_turn_steer_event_params(&tracking, turn_steer),
}));
}
fn maybe_emit_turn_event(&mut self, turn_id: &str, out: &mut Vec<TrackEventRequest>) {
let Some(turn_state) = self.turns.get(turn_id) else {
return;
@@ -573,7 +603,7 @@ fn codex_turn_event_params(
is_first_turn,
status: completed.status,
turn_error: completed.turn_error,
steer_count: None,
steer_count: Some(turn_state.steer_count),
total_tool_call_count: None,
shell_command_count: None,
file_change_count: None,