mirror of
https://github.com/openai/codex.git
synced 2026-05-02 12:21:26 +03:00
5ea107a088
Took over the work that @aaronl-openai started here: https://github.com/openai/codex/pull/10397 Now that app-server clients are able to set up custom tools (called `dynamic_tools` in app-server), we should expose a way for clients to pass in not just text, but also image outputs. This is something the Responses API already supports for function call outputs, where you can pass in either a string or an array of content outputs (text, image, file): https://platform.openai.com/docs/api-reference/responses/create#responses_create-input-input_item_list-item-function_tool_call_output-output-array-input_image So let's just plumb it through in Codex (with the caveat that we only support text and image for now). This is implemented end-to-end across app-server v2 protocol types and core tool handling. ## Breaking API change NOTE: This introduces a breaking change with dynamic tools, but I think it's ok since this concept was only recently introduced (https://github.com/openai/codex/pull/9539) and it's better to get the API contract correct. I don't think there are any real consumers of this yet (not even the Codex App). Old shape: `{ "output": "dynamic-ok", "success": true }` New shape: ``` { "contentItems": [ { "type": "inputText", "text": "dynamic-ok" }, { "type": "inputImage", "imageUrl": "data:image/png;base64,AAA" } ] "success": true } ```
72 lines
2.3 KiB
Rust
72 lines
2.3 KiB
Rust
use codex_app_server_protocol::DynamicToolCallResponse;
|
|
use codex_core::CodexThread;
|
|
use codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem as CoreDynamicToolCallOutputContentItem;
|
|
use codex_protocol::dynamic_tools::DynamicToolResponse as CoreDynamicToolResponse;
|
|
use codex_protocol::protocol::Op;
|
|
use std::sync::Arc;
|
|
use tokio::sync::oneshot;
|
|
use tracing::error;
|
|
|
|
pub(crate) async fn on_call_response(
|
|
call_id: String,
|
|
receiver: oneshot::Receiver<serde_json::Value>,
|
|
conversation: Arc<CodexThread>,
|
|
) {
|
|
let response = receiver.await;
|
|
let value = match response {
|
|
Ok(value) => value,
|
|
Err(err) => {
|
|
error!("request failed: {err:?}");
|
|
let fallback = CoreDynamicToolResponse {
|
|
content_items: vec![CoreDynamicToolCallOutputContentItem::InputText {
|
|
text: "dynamic tool request failed".to_string(),
|
|
}],
|
|
success: false,
|
|
};
|
|
if let Err(err) = conversation
|
|
.submit(Op::DynamicToolResponse {
|
|
id: call_id.clone(),
|
|
response: fallback,
|
|
})
|
|
.await
|
|
{
|
|
error!("failed to submit DynamicToolResponse: {err}");
|
|
}
|
|
return;
|
|
}
|
|
};
|
|
|
|
let response = serde_json::from_value::<DynamicToolCallResponse>(value).unwrap_or_else(|err| {
|
|
error!("failed to deserialize DynamicToolCallResponse: {err}");
|
|
DynamicToolCallResponse {
|
|
content_items: vec![
|
|
codex_app_server_protocol::DynamicToolCallOutputContentItem::InputText {
|
|
text: "dynamic tool response was invalid".to_string(),
|
|
},
|
|
],
|
|
success: false,
|
|
}
|
|
});
|
|
|
|
let DynamicToolCallResponse {
|
|
content_items,
|
|
success,
|
|
} = response;
|
|
let response = CoreDynamicToolResponse {
|
|
content_items: content_items
|
|
.into_iter()
|
|
.map(CoreDynamicToolCallOutputContentItem::from)
|
|
.collect(),
|
|
success,
|
|
};
|
|
if let Err(err) = conversation
|
|
.submit(Op::DynamicToolResponse {
|
|
id: call_id,
|
|
response,
|
|
})
|
|
.await
|
|
{
|
|
error!("failed to submit DynamicToolResponse: {err}");
|
|
}
|
|
}
|