mirrors/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-05-05 13:51:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

realtime: use codex tool handoff and fix audio playback

Browse Source
This commit is contained in:
Ahmed Ibrahim
2026-03-04 16:55:47 -08:00
parent 8a59386273
commit f13917d50e
8 changed files with 667 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files

258
codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs
View File

@@ -8,6 +8,11 @@ use crate::endpoint::realtime_websocket::protocol::SessionAudio;
use crate::endpoint::realtime_websocket::protocol::SessionAudioFormat;
use crate::endpoint::realtime_websocket::protocol::SessionAudioInput;
use crate::endpoint::realtime_websocket::protocol::SessionAudioOutput;
use crate::endpoint::realtime_websocket::protocol::SessionAudioOutputFormat;
use crate::endpoint::realtime_websocket::protocol::SessionTool;
use crate::endpoint::realtime_websocket::protocol::SessionToolParameters;
use crate::endpoint::realtime_websocket::protocol::SessionToolProperties;
use crate::endpoint::realtime_websocket::protocol::SessionToolProperty;
use crate::endpoint::realtime_websocket::protocol::SessionUpdateSession;
use crate::endpoint::realtime_websocket::protocol::parse_realtime_event;
use crate::error::ApiError;
@@ -17,6 +22,7 @@ use futures::SinkExt;
use futures::StreamExt;
use http::HeaderMap;
use http::HeaderValue;
use serde_json::json;
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
@@ -220,6 +226,20 @@ impl RealtimeWebsocketConnection {
.await
}
pub async fn send_function_call_output(
&self,
call_id: String,
output_text: String,
) -> Result<(), ApiError> {
self.writer
.send_function_call_output(call_id, output_text)
.await
}
pub async fn send_response_create(&self) -> Result<(), ApiError> {
self.writer.send_response_create().await
}
pub async fn close(&self) -> Result<(), ApiError> {
self.writer.close().await
}
@@ -263,11 +283,10 @@ impl RealtimeWebsocketWriter {
pub async fn send_conversation_item_create(&self, text: String) -> Result<(), ApiError> {
self.send_json(RealtimeOutboundMessage::ConversationItemCreate {
item: ConversationItem {
kind: "message".to_string(),
item: ConversationItem::Message {
role: "user".to_string(),
content: vec![ConversationItemContent {
kind: "text".to_string(),
kind: "input_text".to_string(),
text,
}],
},
@@ -277,21 +296,47 @@ impl RealtimeWebsocketWriter {
pub async fn send_conversation_handoff_append(
&self,
handoff_id: String,
_handoff_id: String,
output_text: String,
) -> Result<(), ApiError> {
self.send_json(RealtimeOutboundMessage::ConversationHandoffAppend {
handoff_id,
output_text,
self.send_json(RealtimeOutboundMessage::ConversationItemCreate {
item: ConversationItem::Message {
role: "assistant".to_string(),
content: vec![ConversationItemContent {
kind: "output_text".to_string(),
text: output_text,
}],
},
})
.await
}
pub async fn send_function_call_output(
&self,
call_id: String,
output_text: String,
) -> Result<(), ApiError> {
let output = json!({
"content": output_text,
})
.to_string();
self.send_json(RealtimeOutboundMessage::ConversationItemCreate {
item: ConversationItem::FunctionCallOutput { call_id, output },
})
.await
}
pub async fn send_response_create(&self) -> Result<(), ApiError> {
self.send_json(RealtimeOutboundMessage::ResponseCreate)
.await
}
pub async fn send_session_update(&self, instructions: String) -> Result<(), ApiError> {
self.send_json(RealtimeOutboundMessage::SessionUpdate {
session: SessionUpdateSession {
kind: "quicksilver".to_string(),
kind: "realtime".to_string(),
instructions,
output_modalities: vec!["audio".to_string()],
audio: SessionAudio {
input: SessionAudioInput {
format: SessionAudioFormat {
@@ -300,9 +345,30 @@ impl RealtimeWebsocketWriter {
},
},
output: SessionAudioOutput {
voice: "mundo".to_string(),
format: SessionAudioOutputFormat {
kind: "audio/pcm".to_string(),
},
voice: "marin".to_string(),
},
},
tools: vec![SessionTool {
kind: "function".to_string(),
name: "codex".to_string(),
description:
"Delegate a request to Codex and return the final result to the user."
.to_string(),
parameters: SessionToolParameters {
kind: "object".to_string(),
properties: SessionToolProperties {
prompt: SessionToolProperty {
kind: "string".to_string(),
description: "The user request to delegate to Codex.".to_string(),
},
},
required: vec!["prompt".to_string()],
},
}],
tool_choice: "auto".to_string(),
},
})
.await
@@ -510,15 +576,16 @@ fn websocket_url_from_api_url(
}
}
{
let has_additional_query_params = query_params
.is_some_and(|params| params.keys().any(|key| key != "model" || model.is_none()));
if model.is_some() || has_additional_query_params {
let mut query = url.query_pairs_mut();
query.append_pair("intent", "quicksilver");
if let Some(model) = model {
query.append_pair("model", model);
}
if let Some(query_params) = query_params {
for (key, value) in query_params {
if key == "intent" || (key == "model" && model.is_some()) {
if key == "model" && model.is_some() {
continue;
}
query.append_pair(key, value);
@@ -590,7 +657,7 @@ mod tests {
#[test]
fn parse_audio_delta_event() {
let payload = json!({
"type": "conversation.output_audio.delta",
"type": "response.output_audio.delta",
"delta": "AAA=",
"sample_rate": 48000,
"channels": 1,
@@ -608,6 +675,25 @@ mod tests {
);
}
#[test]
fn parse_audio_delta_event_defaults_audio_shape() {
let payload = json!({
"type": "response.output_audio.delta",
"delta": "AAA="
})
.to_string();
assert_eq!(
parse_realtime_event(payload.as_str()),
Some(RealtimeEvent::AudioOut(RealtimeAudioFrame {
data: "AAA=".to_string(),
sample_rate: 24_000,
num_channels: 1,
samples_per_channel: None,
}))
);
}
#[test]
fn parse_conversation_item_added_event() {
let payload = json!({
@@ -641,13 +727,18 @@ mod tests {
#[test]
fn parse_handoff_requested_event() {
let payload = json!({
"type": "conversation.handoff.requested",
"handoff_id": "handoff_123",
"item_id": "item_123",
"input_transcript": "delegate this",
"messages": [
{"role": "user", "text": "delegate this"}
]
"type": "response.done",
"response": {
"output": [
{
"id": "item_123",
"type": "function_call",
"name": "codex",
"call_id": "handoff_123",
"arguments": "{\"prompt\":\"delegate this\"}"
}
]
}
})
.to_string();
@@ -665,6 +756,24 @@ mod tests {
);
}
#[test]
fn parse_unknown_event_as_conversation_item_added() {
let payload = json!({
"type": "response.output_text.delta",
"delta": "hello",
"response_id": "resp_1",
})
.to_string();
assert_eq!(
parse_realtime_event(payload.as_str()),
Some(RealtimeEvent::ConversationItemAdded(json!({
"type": "response.output_text.delta",
"delta": "hello",
"response_id": "resp_1",
})))
);
}
#[test]
fn merge_request_headers_matches_http_precedence() {
let mut provider_headers = HeaderMap::new();
@@ -702,10 +811,7 @@ mod tests {
fn websocket_url_from_http_base_defaults_to_ws_path() {
let url =
websocket_url_from_api_url("http://127.0.0.1:8011", None, None).expect("build ws url");
assert_eq!(
url.as_str(),
"ws://127.0.0.1:8011/v1/realtime?intent=quicksilver"
);
assert_eq!(url.as_str(), "ws://127.0.0.1:8011/v1/realtime");
}
#[test]
@@ -715,7 +821,7 @@ mod tests {
.expect("build ws url");
assert_eq!(
url.as_str(),
"wss://example.com/v1/realtime?intent=quicksilver&model=realtime-test-model"
"wss://example.com/v1/realtime?model=realtime-test-model"
);
}
@@ -725,7 +831,7 @@ mod tests {
.expect("build ws url");
assert_eq!(
url.as_str(),
"wss://api.openai.com/v1/realtime?intent=quicksilver&model=snapshot"
"wss://api.openai.com/v1/realtime?model=snapshot"
);
}
@@ -736,7 +842,7 @@ mod tests {
.expect("build ws url");
assert_eq!(
url.as_str(),
"wss://example.com/openai/v1/realtime?intent=quicksilver&model=snapshot"
"wss://example.com/openai/v1/realtime?model=snapshot"
);
}
@@ -744,16 +850,13 @@ mod tests {
fn websocket_url_preserves_existing_realtime_path_and_extra_query_params() {
let url = websocket_url_from_api_url(
"https://example.com/v1/realtime?foo=bar",
Some(&HashMap::from([
("trace".to_string(), "1".to_string()),
("intent".to_string(), "ignored".to_string()),
])),
Some(&HashMap::from([("trace".to_string(), "1".to_string())])),
Some("snapshot"),
)
.expect("build ws url");
assert_eq!(
url.as_str(),
"wss://example.com/v1/realtime?foo=bar&intent=quicksilver&model=snapshot&trace=1"
"wss://example.com/v1/realtime?foo=bar&model=snapshot&trace=1"
);
}
@@ -777,12 +880,16 @@ mod tests {
assert_eq!(first_json["type"], "session.update");
assert_eq!(
first_json["session"]["type"],
Value::String("quicksilver".to_string())
Value::String("realtime".to_string())
);
assert_eq!(
first_json["session"]["instructions"],
Value::String("backend prompt".to_string())
);
assert_eq!(
first_json["session"]["output_modalities"][0],
Value::String("audio".to_string())
);
assert_eq!(
first_json["session"]["audio"]["input"]["format"]["type"],
Value::String("audio/pcm".to_string())
@@ -791,9 +898,29 @@ mod tests {
first_json["session"]["audio"]["input"]["format"]["rate"],
Value::from(24_000)
);
assert_eq!(
first_json["session"]["audio"]["output"]["format"]["type"],
Value::String("audio/pcm".to_string())
);
assert_eq!(
first_json["session"]["audio"]["output"]["voice"],
Value::String("mundo".to_string())
Value::String("marin".to_string())
);
assert_eq!(
first_json["session"]["tool_choice"],
Value::String("auto".to_string())
);
assert_eq!(
first_json["session"]["tools"][0]["type"],
Value::String("function".to_string())
);
assert_eq!(
first_json["session"]["tools"][0]["name"],
Value::String("codex".to_string())
);
assert_eq!(
first_json["session"]["tools"][0]["parameters"]["required"][0],
Value::String("prompt".to_string())
);
ws.send(Message::Text(
@@ -836,13 +963,43 @@ mod tests {
.into_text()
.expect("text");
let fourth_json: Value = serde_json::from_str(&fourth).expect("json");
assert_eq!(fourth_json["type"], "conversation.handoff.append");
assert_eq!(fourth_json["handoff_id"], "handoff_1");
assert_eq!(fourth_json["output_text"], "hello from codex");
assert_eq!(fourth_json["type"], "conversation.item.create");
assert_eq!(fourth_json["item"]["type"], "message");
assert_eq!(fourth_json["item"]["role"], "assistant");
assert_eq!(
fourth_json["item"]["content"][0]["type"],
Value::String("output_text".to_string())
);
assert_eq!(
fourth_json["item"]["content"][0]["text"],
Value::String("hello from codex".to_string())
);
let fifth = ws
.next()
.await
.expect("fifth msg")
.expect("fifth msg ok")
.into_text()
.expect("text");
let fifth_json: Value = serde_json::from_str(&fifth).expect("json");
assert_eq!(fifth_json["type"], "conversation.item.create");
assert_eq!(fifth_json["item"]["type"], "function_call_output");
assert_eq!(fifth_json["item"]["call_id"], "handoff_1");
let sixth = ws
.next()
.await
.expect("sixth msg")
.expect("sixth msg ok")
.into_text()
.expect("text");
let sixth_json: Value = serde_json::from_str(&sixth).expect("json");
assert_eq!(sixth_json["type"], "response.create");
ws.send(Message::Text(
json!({
"type": "conversation.output_audio.delta",
"type": "response.output_audio.delta",
"delta": "AQID",
"sample_rate": 48000,
"channels": 1
@@ -855,11 +1012,18 @@ mod tests {
ws.send(Message::Text(
json!({
"type": "conversation.handoff.requested",
"handoff_id": "handoff_1",
"item_id": "item_2",
"input_transcript": "delegate now",
"messages": [{"role": "user", "text": "delegate now"}]
"type": "response.done",
"response": {
"output": [
{
"id": "item_2",
"type": "function_call",
"name": "codex",
"call_id": "handoff_1",
"arguments": "{\"prompt\":\"delegate now\"}"
}
]
}
})
.to_string()
.into(),
@@ -929,6 +1093,14 @@ mod tests {
)
.await
.expect("send handoff");
connection
.send_function_call_output("handoff_1".to_string(), "final from codex".to_string())
.await
.expect("send function output");
connection
.send_response_create()
.await
.expect("send response.create");
let audio_event = connection
.next_event()

201
codex-rs/codex-api/src/endpoint/realtime_websocket/protocol.rs
View File

@@ -2,8 +2,10 @@ pub use codex_protocol::protocol::RealtimeAudioFrame;
pub use codex_protocol::protocol::RealtimeEvent;
pub use codex_protocol::protocol::RealtimeHandoffMessage;
pub use codex_protocol::protocol::RealtimeHandoffRequested;
use serde::Deserialize;
use serde::Serialize;
use serde_json::Value;
use std::string::ToString;
use tracing::debug;
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -18,11 +20,8 @@ pub struct RealtimeSessionConfig {
pub(super) enum RealtimeOutboundMessage {
#[serde(rename = "input_audio_buffer.append")]
InputAudioBufferAppend { audio: String },
#[serde(rename = "conversation.handoff.append")]
ConversationHandoffAppend {
handoff_id: String,
output_text: String,
},
#[serde(rename = "response.create")]
ResponseCreate,
#[serde(rename = "session.update")]
SessionUpdate { session: SessionUpdateSession },
#[serde(rename = "conversation.item.create")]
@@ -34,7 +33,10 @@ pub(super) struct SessionUpdateSession {
#[serde(rename = "type")]
pub(super) kind: String,
pub(super) instructions: String,
pub(super) output_modalities: Vec<String>,
pub(super) audio: SessionAudio,
pub(super) tools: Vec<SessionTool>,
pub(super) tool_choice: String,
}
#[derive(Debug, Clone, Serialize)]
@@ -57,15 +59,55 @@ pub(super) struct SessionAudioFormat {
#[derive(Debug, Clone, Serialize)]
pub(super) struct SessionAudioOutput {
pub(super) format: SessionAudioOutputFormat,
pub(super) voice: String,
}
#[derive(Debug, Clone, Serialize)]
pub(super) struct ConversationItem {
pub(super) struct SessionAudioOutputFormat {
#[serde(rename = "type")]
pub(super) kind: String,
pub(super) role: String,
pub(super) content: Vec<ConversationItemContent>,
}
#[derive(Debug, Clone, Serialize)]
pub(super) struct SessionTool {
#[serde(rename = "type")]
pub(super) kind: String,
pub(super) name: String,
pub(super) description: String,
pub(super) parameters: SessionToolParameters,
}
#[derive(Debug, Clone, Serialize)]
pub(super) struct SessionToolParameters {
#[serde(rename = "type")]
pub(super) kind: String,
pub(super) properties: SessionToolProperties,
pub(super) required: Vec<String>,
}
#[derive(Debug, Clone, Serialize)]
pub(super) struct SessionToolProperties {
pub(super) prompt: SessionToolProperty,
}
#[derive(Debug, Clone, Serialize)]
pub(super) struct SessionToolProperty {
#[serde(rename = "type")]
pub(super) kind: String,
pub(super) description: String,
}
#[derive(Debug, Clone, Serialize)]
#[serde(tag = "type")]
pub(super) enum ConversationItem {
#[serde(rename = "message")]
Message {
role: String,
content: Vec<ConversationItemContent>,
},
#[serde(rename = "function_call_output")]
FunctionCallOutput { call_id: String, output: String },
}
#[derive(Debug, Clone, Serialize)]
@@ -92,7 +134,7 @@ pub(super) fn parse_realtime_event(payload: &str) -> Option<RealtimeEvent> {
}
};
match message_type {
"session.updated" => {
"session.created" | "session.updated" => {
let session_id = parsed
.get("session")
.and_then(Value::as_object)
@@ -110,7 +152,7 @@ pub(super) fn parse_realtime_event(payload: &str) -> Option<RealtimeEvent> {
instructions,
})
}
"conversation.output_audio.delta" => {
"response.output_audio.delta" => {
let data = parsed
.get("delta")
.and_then(Value::as_str)
@@ -119,12 +161,14 @@ pub(super) fn parse_realtime_event(payload: &str) -> Option<RealtimeEvent> {
let sample_rate = parsed
.get("sample_rate")
.and_then(Value::as_u64)
.and_then(|v| u32::try_from(v).ok())?;
.and_then(|v| u32::try_from(v).ok())
.unwrap_or(24_000);
let num_channels = parsed
.get("channels")
.or_else(|| parsed.get("num_channels"))
.and_then(Value::as_u64)
.and_then(|v| u16::try_from(v).ok())?;
.and_then(|v| u16::try_from(v).ok())
.unwrap_or(1);
Some(RealtimeEvent::AudioOut(RealtimeAudioFrame {
data,
sample_rate,
@@ -146,35 +190,11 @@ pub(super) fn parse_realtime_event(payload: &str) -> Option<RealtimeEvent> {
.and_then(Value::as_str)
.map(str::to_string)
.map(|item_id| RealtimeEvent::ConversationItemDone { item_id }),
"conversation.handoff.requested" => {
let handoff_id = parsed
.get("handoff_id")
.and_then(Value::as_str)
.map(str::to_string)?;
let item_id = parsed
.get("item_id")
.and_then(Value::as_str)
.map(str::to_string)?;
let input_transcript = parsed
.get("input_transcript")
.and_then(Value::as_str)
.map(str::to_string)?;
let messages = parsed
.get("messages")
.and_then(Value::as_array)?
.iter()
.filter_map(|message| {
let role = message.get("role").and_then(Value::as_str)?.to_string();
let text = message.get("text").and_then(Value::as_str)?.to_string();
Some(RealtimeHandoffMessage { role, text })
})
.collect();
Some(RealtimeEvent::HandoffRequested(RealtimeHandoffRequested {
handoff_id,
item_id,
input_transcript,
messages,
}))
"response.done" => {
if let Some(handoff) = parse_handoff_requested(&parsed) {
return Some(RealtimeEvent::HandoffRequested(handoff));
}
Some(RealtimeEvent::ConversationItemAdded(parsed))
}
"error" => parsed
.get("message")
@@ -188,11 +208,100 @@ pub(super) fn parse_realtime_event(payload: &str) -> Option<RealtimeEvent> {
.and_then(Value::as_str)
.map(str::to_string)
})
.or_else(|| parsed.get("error").map(std::string::ToString::to_string))
.or_else(|| parsed.get("error").map(ToString::to_string))
.map(RealtimeEvent::Error),
_ => {
debug!("received unsupported realtime event type: {message_type}, data: {payload}");
None
}
_ => Some(RealtimeEvent::ConversationItemAdded(parsed)),
}
}
fn parse_handoff_requested(parsed: &Value) -> Option<RealtimeHandoffRequested> {
let outputs = parsed
.get("response")
.and_then(Value::as_object)
.and_then(|response| response.get("output"))
.and_then(Value::as_array)?;
let function_call = outputs.iter().find(|item| {
item.get("type").and_then(Value::as_str) == Some("function_call")
&& item.get("name").and_then(Value::as_str) == Some("codex")
})?;
let handoff_id = function_call
.get("call_id")
.and_then(Value::as_str)
.map(str::to_string)?;
let item_id = function_call
.get("id")
.and_then(Value::as_str)
.map(str::to_string)
.unwrap_or_else(|| handoff_id.clone());
let arguments = function_call
.get("arguments")
.and_then(Value::as_str)
.unwrap_or_default();
let (input_transcript, messages) = parse_handoff_arguments(arguments);
Some(RealtimeHandoffRequested {
handoff_id,
item_id,
input_transcript,
messages,
})
}
fn parse_handoff_arguments(arguments: &str) -> (String, Vec<RealtimeHandoffMessage>) {
#[derive(Debug, Deserialize)]
struct HandoffArguments {
#[serde(default)]
prompt: Option<String>,
#[serde(default)]
text: Option<String>,
#[serde(default)]
input: Option<String>,
#[serde(default)]
message: Option<String>,
#[serde(default)]
input_transcript: Option<String>,
#[serde(default)]
messages: Vec<RealtimeHandoffMessage>,
}
let Some(parsed) = serde_json::from_str::<HandoffArguments>(arguments).ok() else {
return (
arguments.to_string(),
vec![RealtimeHandoffMessage {
role: "user".to_string(),
text: arguments.to_string(),
}],
);
};
let messages = parsed
.messages
.into_iter()
.filter(|message| !message.text.is_empty())
.collect::<Vec<_>>();
for value in [
parsed.prompt,
parsed.text,
parsed.input,
parsed.message,
parsed.input_transcript,
]
.into_iter()
.flatten()
{
if !value.is_empty() {
if messages.is_empty() {
return (
value.clone(),
vec![RealtimeHandoffMessage {
role: "user".to_string(),
text: value,
}],
);
}
return (value, messages);
}
}
if let Some(first_message) = messages.first() {
return (first_message.text.clone(), messages);
}
(String::new(), messages)
}