mirror of
https://github.com/openai/codex.git
synced 2026-05-02 04:11:39 +03:00
Support multimodal custom tool outputs (#12948)
## Summary This changes `custom_tool_call_output` to use the same output payload shape as `function_call_output`, so freeform tools can return either plain text or structured content items. The main goal is to let `js_repl` return image content from nested `view_image` calls in its own `custom_tool_call_output`, instead of relying on a separate injected message. ## What changed - Changed `custom_tool_call_output.output` from `string` to `FunctionCallOutputPayload` - Updated freeform tool plumbing to preserve structured output bodies - Updated `js_repl` to aggregate nested tool content items and attach them to the outer `js_repl` result - Removed the old `js_repl` special case that injected `view_image` results as a separate pending user image message - Updated normalization/history/truncation paths to handle multimodal `custom_tool_call_output` - Regenerated app-server protocol schema artifacts ## Behavior Direct `view_image` calls still return a `function_call_output` with image content. When `view_image` is called inside `js_repl`, the outer `js_repl` `custom_tool_call_output` now carries: - an `input_text` item if the JS produced text output - one or more `input_image` items from nested tool results So the nested image result now stays inside the `js_repl` tool output instead of being injected as a separate message. ## Compatibility This is intended to be backward-compatible for resumed conversations. Older histories that stored `custom_tool_call_output.output` as a plain string still deserialize correctly, and older histories that used the previous injected-image-message flow also continue to resume. Added regression coverage for resuming a pre-change rollout containing: - string-valued `custom_tool_call_output` - legacy injected image message history #### [git stack](https://github.com/magus/git-stack-cli) - 👉 `1` https://github.com/openai/codex/pull/12948
This commit is contained in:
committed by
GitHub
parent
f90e97e414
commit
7e980d7db6
@@ -95,15 +95,12 @@ impl ToolOutput {
|
||||
match self {
|
||||
ToolOutput::Function { body, success } => {
|
||||
// `custom_tool_call` is the Responses API item type for freeform
|
||||
// tools (`ToolSpec::Freeform`, e.g. freeform `apply_patch`).
|
||||
// Those payloads must round-trip as `custom_tool_call_output`
|
||||
// with plain string output.
|
||||
// tools (`ToolSpec::Freeform`, e.g. freeform `apply_patch` or
|
||||
// `js_repl`).
|
||||
if matches!(payload, ToolPayload::Custom { .. }) {
|
||||
// Freeform/custom tools (`custom_tool_call`) use the custom
|
||||
// output wire shape and remain string-only.
|
||||
return ResponseInputItem::CustomToolCallOutput {
|
||||
call_id: call_id.to_string(),
|
||||
output: body.to_text().unwrap_or_default(),
|
||||
output: FunctionCallOutputPayload { body, success },
|
||||
};
|
||||
}
|
||||
|
||||
@@ -183,7 +180,9 @@ mod tests {
|
||||
match response {
|
||||
ResponseInputItem::CustomToolCallOutput { call_id, output } => {
|
||||
assert_eq!(call_id, "call-42");
|
||||
assert_eq!(output, "patched");
|
||||
assert_eq!(output.text_content(), Some("patched"));
|
||||
assert!(output.content_items().is_none());
|
||||
assert_eq!(output.success, Some(true));
|
||||
}
|
||||
other => panic!("expected CustomToolCallOutput, got {other:?}"),
|
||||
}
|
||||
@@ -234,8 +233,21 @@ mod tests {
|
||||
|
||||
match response {
|
||||
ResponseInputItem::CustomToolCallOutput { call_id, output } => {
|
||||
let expected = vec![
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "line 1".to_string(),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,AAA".to_string(),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "line 2".to_string(),
|
||||
},
|
||||
];
|
||||
assert_eq!(call_id, "call-99");
|
||||
assert_eq!(output, "line 1\nline 2");
|
||||
assert_eq!(output.content_items(), Some(expected.as_slice()));
|
||||
assert_eq!(output.body.to_text().as_deref(), Some("line 1\nline 2"));
|
||||
assert_eq!(output.success, Some(true));
|
||||
}
|
||||
other => panic!("expected CustomToolCallOutput, got {other:?}"),
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user