mirror of
https://github.com/openai/codex.git
synced 2026-04-21 15:01:50 +03:00
Compare commits
1 Commits
dev/shaqay
...
fjord/orig
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
dc1949f3c4 |
2
codex-rs/Cargo.lock
generated
2
codex-rs/Cargo.lock
generated
@@ -1775,6 +1775,7 @@ dependencies = [
|
||||
"codex-utils-absolute-path",
|
||||
"codex-utils-cargo-bin",
|
||||
"codex-utils-home-dir",
|
||||
"codex-utils-image",
|
||||
"codex-utils-pty",
|
||||
"codex-utils-readiness",
|
||||
"codex-utils-stream-parser",
|
||||
@@ -1799,6 +1800,7 @@ dependencies = [
|
||||
"landlock",
|
||||
"libc",
|
||||
"maplit",
|
||||
"mime_guess",
|
||||
"notify",
|
||||
"once_cell",
|
||||
"openssl-sys",
|
||||
|
||||
@@ -399,7 +399,10 @@ async fn dynamic_tool_call_round_trip_sends_content_items_to_model() -> Result<(
|
||||
FunctionCallOutputContentItem::InputText { text }
|
||||
}
|
||||
DynamicToolCallOutputContentItem::InputImage { image_url } => {
|
||||
FunctionCallOutputContentItem::InputImage { image_url }
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url,
|
||||
detail: None,
|
||||
}
|
||||
}
|
||||
})
|
||||
.collect::<Vec<FunctionCallOutputContentItem>>();
|
||||
|
||||
@@ -46,6 +46,7 @@ codex-protocol = { workspace = true }
|
||||
codex-rmcp-client = { workspace = true }
|
||||
codex-state = { workspace = true }
|
||||
codex-utils-absolute-path = { workspace = true }
|
||||
codex-utils-image = { workspace = true }
|
||||
codex-utils-home-dir = { workspace = true }
|
||||
codex-utils-pty = { workspace = true }
|
||||
codex-utils-readiness = { workspace = true }
|
||||
@@ -62,9 +63,11 @@ eventsource-stream = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
http = { workspace = true }
|
||||
iana-time-zone = { workspace = true }
|
||||
image = { workspace = true, features = ["jpeg", "png"] }
|
||||
indexmap = { workspace = true }
|
||||
keyring = { workspace = true, features = ["crypto-rust"] }
|
||||
libc = { workspace = true }
|
||||
mime_guess = { workspace = true }
|
||||
notify = { workspace = true }
|
||||
once_cell = { workspace = true }
|
||||
os_info = { workspace = true }
|
||||
@@ -155,7 +158,6 @@ codex-test-macros = { workspace = true }
|
||||
codex-utils-cargo-bin = { workspace = true }
|
||||
core_test_support = { workspace = true }
|
||||
ctor = { workspace = true }
|
||||
image = { workspace = true, features = ["jpeg", "png"] }
|
||||
insta = { workspace = true }
|
||||
maplit = { workspace = true }
|
||||
predicates = { workspace = true }
|
||||
|
||||
@@ -430,6 +430,9 @@
|
||||
"use_linux_sandbox_bwrap": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"view_image_original_resolution": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"voice_transcription": {
|
||||
"type": "boolean"
|
||||
},
|
||||
@@ -1761,6 +1764,9 @@
|
||||
"use_linux_sandbox_bwrap": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"view_image_original_resolution": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"voice_transcription": {
|
||||
"type": "boolean"
|
||||
},
|
||||
|
||||
@@ -522,7 +522,7 @@ impl ModelClientSession {
|
||||
summary: ReasoningSummaryConfig,
|
||||
) -> Result<ResponsesApiRequest> {
|
||||
let instructions = &prompt.base_instructions.text;
|
||||
let input = prompt.get_formatted_input();
|
||||
let input = prompt.get_formatted_input()?;
|
||||
let tools = create_tools_json_for_responses_api(&prompt.tools)?;
|
||||
let default_reasoning_effort = model_info.default_reasoning_level;
|
||||
let reasoning = if model_info.supports_reasoning_summaries {
|
||||
|
||||
@@ -1,9 +1,14 @@
|
||||
use crate::client_common::tools::ToolSpec;
|
||||
use crate::config::types::Personality;
|
||||
use crate::error::CodexErr;
|
||||
use crate::error::Result;
|
||||
use base64::Engine as _;
|
||||
use base64::prelude::BASE64_STANDARD;
|
||||
pub use codex_api::common::ResponseEvent;
|
||||
use codex_protocol::models::BaseInstructions;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputBody;
|
||||
use codex_protocol::models::FunctionCallOutputContentItem;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use futures::Stream;
|
||||
use serde::Deserialize;
|
||||
@@ -22,6 +27,12 @@ pub const REVIEW_EXIT_SUCCESS_TMPL: &str = include_str!("../templates/review/exi
|
||||
pub const REVIEW_EXIT_INTERRUPTED_TMPL: &str =
|
||||
include_str!("../templates/review/exit_interrupted.xml");
|
||||
|
||||
// See the Responses API image input size limits in the Images and Vision guide:
|
||||
// https://platform.openai.com/docs/guides/images-vision?api-mode=responses&format=file
|
||||
const RESPONSES_API_MAX_INLINE_IMAGE_BYTES: usize = 50_000_000;
|
||||
const RESPONSES_API_MAX_INLINE_IMAGE_BYTES_LABEL: &str = "50 MB";
|
||||
const INLINE_TOOL_IMAGE_OMITTED_PLACEHOLDER: &str = "Codex omitted this tool-returned image because the current request would exceed the Responses API 50 MB total image limit. Request fewer images at a time or inspect them in smaller batches.";
|
||||
|
||||
/// API request payload for a single model turn
|
||||
#[derive(Default, Debug, Clone)]
|
||||
pub struct Prompt {
|
||||
@@ -45,7 +56,7 @@ pub struct Prompt {
|
||||
}
|
||||
|
||||
impl Prompt {
|
||||
pub(crate) fn get_formatted_input(&self) -> Vec<ResponseItem> {
|
||||
pub(crate) fn get_formatted_input(&self) -> Result<Vec<ResponseItem>> {
|
||||
let mut input = self.input.clone();
|
||||
|
||||
// when using the *Freeform* apply_patch tool specifically, tool outputs
|
||||
@@ -60,7 +71,156 @@ impl Prompt {
|
||||
reserialize_shell_outputs(&mut input);
|
||||
}
|
||||
|
||||
input
|
||||
enforce_inline_image_request_budget(&mut input, RESPONSES_API_MAX_INLINE_IMAGE_BYTES)?;
|
||||
|
||||
Ok(input)
|
||||
}
|
||||
}
|
||||
|
||||
fn enforce_inline_image_request_budget(
|
||||
items: &mut [ResponseItem],
|
||||
max_inline_image_bytes: usize,
|
||||
) -> Result<()> {
|
||||
let mut inline_image_bytes = total_inline_image_bytes(items);
|
||||
let mut omitted_model_generated_image = false;
|
||||
|
||||
if inline_image_bytes <= max_inline_image_bytes {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
for item in items.iter_mut().rev() {
|
||||
if inline_image_bytes <= max_inline_image_bytes {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let Some(content_items) = tool_output_content_items_mut(item) else {
|
||||
continue;
|
||||
};
|
||||
|
||||
for content_item in content_items.iter_mut().rev() {
|
||||
if inline_image_bytes <= max_inline_image_bytes {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let FunctionCallOutputContentItem::InputImage { image_url, .. } = content_item else {
|
||||
continue;
|
||||
};
|
||||
let Some(image_bytes) = inline_image_data_url_bytes(image_url) else {
|
||||
continue;
|
||||
};
|
||||
|
||||
*content_item = FunctionCallOutputContentItem::InputText {
|
||||
text: INLINE_TOOL_IMAGE_OMITTED_PLACEHOLDER.to_string(),
|
||||
};
|
||||
inline_image_bytes = inline_image_bytes.saturating_sub(image_bytes);
|
||||
omitted_model_generated_image = true;
|
||||
}
|
||||
}
|
||||
|
||||
Err(CodexErr::InvalidRequest(
|
||||
inline_image_request_budget_exceeded_message(
|
||||
inline_image_bytes,
|
||||
max_inline_image_bytes,
|
||||
omitted_model_generated_image,
|
||||
),
|
||||
))
|
||||
}
|
||||
|
||||
fn total_inline_image_bytes(items: &[ResponseItem]) -> usize {
|
||||
items
|
||||
.iter()
|
||||
.map(response_item_inline_image_bytes)
|
||||
.sum::<usize>()
|
||||
}
|
||||
|
||||
fn response_item_inline_image_bytes(item: &ResponseItem) -> usize {
|
||||
match item {
|
||||
ResponseItem::Message { content, .. } => content
|
||||
.iter()
|
||||
.filter_map(|content_item| match content_item {
|
||||
ContentItem::InputImage { image_url } => inline_image_data_url_bytes(image_url),
|
||||
ContentItem::InputText { .. } | ContentItem::OutputText { .. } => None,
|
||||
})
|
||||
.sum::<usize>(),
|
||||
ResponseItem::FunctionCallOutput { output, .. }
|
||||
| ResponseItem::CustomToolCallOutput { output, .. } => output
|
||||
.content_items()
|
||||
.map(|content_items| {
|
||||
content_items
|
||||
.iter()
|
||||
.filter_map(|content_item| match content_item {
|
||||
FunctionCallOutputContentItem::InputImage { image_url, .. } => {
|
||||
inline_image_data_url_bytes(image_url)
|
||||
}
|
||||
FunctionCallOutputContentItem::InputText { .. } => None,
|
||||
})
|
||||
.sum::<usize>()
|
||||
})
|
||||
.unwrap_or_default(),
|
||||
_ => 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn tool_output_content_items_mut(
|
||||
item: &mut ResponseItem,
|
||||
) -> Option<&mut Vec<FunctionCallOutputContentItem>> {
|
||||
match item {
|
||||
ResponseItem::FunctionCallOutput { output, .. }
|
||||
| ResponseItem::CustomToolCallOutput { output, .. } => output.content_items_mut(),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn inline_image_data_url_bytes(url: &str) -> Option<usize> {
|
||||
let payload = parse_base64_image_data_url(url)?;
|
||||
Some(BASE64_STANDARD.decode(payload).ok()?.len())
|
||||
}
|
||||
|
||||
fn parse_base64_image_data_url(url: &str) -> Option<&str> {
|
||||
if !url
|
||||
.get(.."data:".len())
|
||||
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:"))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let comma_index = url.find(',')?;
|
||||
let metadata = &url[..comma_index];
|
||||
let payload = &url[comma_index + 1..];
|
||||
let metadata_without_scheme = &metadata["data:".len()..];
|
||||
let mut metadata_parts = metadata_without_scheme.split(';');
|
||||
let mime_type = metadata_parts.next().unwrap_or_default();
|
||||
let has_base64_marker = metadata_parts.any(|part| part.eq_ignore_ascii_case("base64"));
|
||||
if !mime_type
|
||||
.get(.."image/".len())
|
||||
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("image/"))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
if !has_base64_marker {
|
||||
return None;
|
||||
}
|
||||
Some(payload)
|
||||
}
|
||||
|
||||
fn inline_image_request_budget_exceeded_message(
|
||||
inline_image_bytes: usize,
|
||||
max_inline_image_bytes: usize,
|
||||
omitted_model_generated_image: bool,
|
||||
) -> String {
|
||||
let limit_label = if max_inline_image_bytes == RESPONSES_API_MAX_INLINE_IMAGE_BYTES {
|
||||
RESPONSES_API_MAX_INLINE_IMAGE_BYTES_LABEL.to_string()
|
||||
} else {
|
||||
format!("{max_inline_image_bytes} bytes")
|
||||
};
|
||||
|
||||
if omitted_model_generated_image {
|
||||
format!(
|
||||
"Codex could not send this turn because inline images still total {inline_image_bytes} bytes after omitting all model-generated tool images, exceeding the Responses API {limit_label} total image limit for a single request. Remove some attached images or start a new thread without earlier image attachments."
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
"Codex could not send this turn because inline images total {inline_image_bytes} bytes, exceeding the Responses API {limit_label} total image limit for a single request. Remove some attached images or start a new thread without earlier image attachments."
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -230,10 +390,14 @@ impl Stream for ResponseStream {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use base64::Engine as _;
|
||||
use base64::prelude::BASE64_STANDARD;
|
||||
use codex_api::ResponsesApiRequest;
|
||||
use codex_api::common::OpenAiVerbosity;
|
||||
use codex_api::common::TextControls;
|
||||
use codex_api::create_text_param_for_request;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputPayload;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
@@ -396,4 +560,148 @@ mod tests {
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rewrites_newest_tool_images_until_request_is_within_budget() {
|
||||
let mut items = vec![
|
||||
ResponseItem::Message {
|
||||
id: None,
|
||||
role: "user".to_string(),
|
||||
content: vec![ContentItem::InputImage {
|
||||
image_url: image_data_url(&[1, 2, 3, 4]),
|
||||
}],
|
||||
end_turn: None,
|
||||
phase: None,
|
||||
},
|
||||
ResponseItem::FunctionCallOutput {
|
||||
call_id: "call-1".to_string(),
|
||||
output: FunctionCallOutputPayload::from_content_items(vec![
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: image_data_url(&[5, 6, 7, 8]),
|
||||
detail: None,
|
||||
},
|
||||
]),
|
||||
},
|
||||
ResponseItem::CustomToolCallOutput {
|
||||
call_id: "call-2".to_string(),
|
||||
output: FunctionCallOutputPayload::from_content_items(vec![
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: image_data_url(&[9, 10, 11, 12]),
|
||||
detail: None,
|
||||
},
|
||||
]),
|
||||
},
|
||||
];
|
||||
|
||||
enforce_inline_image_request_budget(&mut items, 8).expect("request should fit");
|
||||
|
||||
assert_eq!(
|
||||
items,
|
||||
vec![
|
||||
ResponseItem::Message {
|
||||
id: None,
|
||||
role: "user".to_string(),
|
||||
content: vec![ContentItem::InputImage {
|
||||
image_url: image_data_url(&[1, 2, 3, 4]),
|
||||
}],
|
||||
end_turn: None,
|
||||
phase: None,
|
||||
},
|
||||
ResponseItem::FunctionCallOutput {
|
||||
call_id: "call-1".to_string(),
|
||||
output: FunctionCallOutputPayload::from_content_items(vec![
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: image_data_url(&[5, 6, 7, 8]),
|
||||
detail: None,
|
||||
},
|
||||
]),
|
||||
},
|
||||
ResponseItem::CustomToolCallOutput {
|
||||
call_id: "call-2".to_string(),
|
||||
output: FunctionCallOutputPayload::from_content_items(vec![
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: INLINE_TOOL_IMAGE_OMITTED_PLACEHOLDER.to_string(),
|
||||
},
|
||||
]),
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn errors_when_user_images_still_exceed_request_budget() {
|
||||
let mut items = vec![ResponseItem::Message {
|
||||
id: None,
|
||||
role: "user".to_string(),
|
||||
content: vec![ContentItem::InputImage {
|
||||
image_url: image_data_url(&[1, 2, 3, 4]),
|
||||
}],
|
||||
end_turn: None,
|
||||
phase: None,
|
||||
}];
|
||||
|
||||
let err = enforce_inline_image_request_budget(&mut items, 3).expect_err("should fail");
|
||||
|
||||
assert_eq!(
|
||||
err.to_string(),
|
||||
"Codex could not send this turn because inline images total 4 bytes, exceeding the Responses API 3 bytes total image limit for a single request. Remove some attached images or start a new thread without earlier image attachments."
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn errors_after_omitting_tool_images_if_user_images_still_exceed_budget() {
|
||||
let mut items = vec![
|
||||
ResponseItem::Message {
|
||||
id: None,
|
||||
role: "user".to_string(),
|
||||
content: vec![ContentItem::InputImage {
|
||||
image_url: image_data_url(&[1, 2, 3, 4]),
|
||||
}],
|
||||
end_turn: None,
|
||||
phase: None,
|
||||
},
|
||||
ResponseItem::FunctionCallOutput {
|
||||
call_id: "call-1".to_string(),
|
||||
output: FunctionCallOutputPayload::from_content_items(vec![
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: image_data_url(&[5, 6, 7, 8]),
|
||||
detail: None,
|
||||
},
|
||||
]),
|
||||
},
|
||||
];
|
||||
|
||||
let err = enforce_inline_image_request_budget(&mut items, 3).expect_err("should fail");
|
||||
|
||||
assert_eq!(
|
||||
err.to_string(),
|
||||
"Codex could not send this turn because inline images still total 4 bytes after omitting all model-generated tool images, exceeding the Responses API 3 bytes total image limit for a single request. Remove some attached images or start a new thread without earlier image attachments."
|
||||
);
|
||||
assert_eq!(
|
||||
items,
|
||||
vec![
|
||||
ResponseItem::Message {
|
||||
id: None,
|
||||
role: "user".to_string(),
|
||||
content: vec![ContentItem::InputImage {
|
||||
image_url: image_data_url(&[1, 2, 3, 4]),
|
||||
}],
|
||||
end_turn: None,
|
||||
phase: None,
|
||||
},
|
||||
ResponseItem::FunctionCallOutput {
|
||||
call_id: "call-1".to_string(),
|
||||
output: FunctionCallOutputPayload::from_content_items(vec![
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: INLINE_TOOL_IMAGE_OMITTED_PLACEHOLDER.to_string(),
|
||||
},
|
||||
]),
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
fn image_data_url(bytes: &[u8]) -> String {
|
||||
format!("data:image/png;base64,{}", BASE64_STANDARD.encode(bytes))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,11 +6,14 @@ use crate::truncate::approx_token_count;
|
||||
use crate::truncate::approx_tokens_from_byte_count_i64;
|
||||
use crate::truncate::truncate_function_output_items_with_policy;
|
||||
use crate::truncate::truncate_text;
|
||||
use base64::Engine;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
|
||||
use codex_protocol::models::BaseInstructions;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputBody;
|
||||
use codex_protocol::models::FunctionCallOutputContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputPayload;
|
||||
use codex_protocol::models::ImageDetail;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::openai_models::InputModality;
|
||||
use codex_protocol::protocol::TokenUsage;
|
||||
@@ -428,7 +431,15 @@ fn estimate_item_token_count(item: &ResponseItem) -> i64 {
|
||||
///
|
||||
/// The estimator later converts bytes to tokens using a 4-bytes/token heuristic
|
||||
/// with ceiling division, so 7,373 bytes maps to approximately 1,844 tokens.
|
||||
const IMAGE_BYTES_ESTIMATE: i64 = 7373;
|
||||
const RESIZED_IMAGE_BYTES_ESTIMATE: i64 = 7373;
|
||||
// See https://developers.openai.com/api/docs/guides/images-vision#calculating-costs.
|
||||
// Use the documented GPT-5 high-detail calculation only for `detail: "original"`;
|
||||
// all other image inputs continue to use `RESIZED_IMAGE_BYTES_ESTIMATE`.
|
||||
const ORIGINAL_IMAGE_MAX_DIM: u32 = 2048;
|
||||
const ORIGINAL_IMAGE_TARGET_SHORT_SIDE: u32 = 768;
|
||||
const ORIGINAL_IMAGE_TILE_SIZE: u32 = 512;
|
||||
const ORIGINAL_IMAGE_BASE_TOKENS: i64 = 70;
|
||||
const ORIGINAL_IMAGE_TILE_TOKENS: i64 = 140;
|
||||
|
||||
pub(crate) fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 {
|
||||
match item {
|
||||
@@ -444,15 +455,15 @@ pub(crate) fn estimate_response_item_model_visible_bytes(item: &ResponseItem) ->
|
||||
let raw = serde_json::to_string(item)
|
||||
.map(|serialized| i64::try_from(serialized.len()).unwrap_or(i64::MAX))
|
||||
.unwrap_or_default();
|
||||
let (payload_bytes, image_count) = image_data_url_estimate_adjustment(item);
|
||||
if payload_bytes == 0 || image_count == 0 {
|
||||
let (payload_bytes, replacement_bytes) = image_data_url_estimate_adjustment(item);
|
||||
if payload_bytes == 0 || replacement_bytes == 0 {
|
||||
raw
|
||||
} else {
|
||||
// Replace raw base64 payload bytes with a fixed per-image cost.
|
||||
// We intentionally preserve the data URL prefix and JSON wrapper
|
||||
// bytes already included in `raw`.
|
||||
// Replace raw base64 payload bytes with a per-image estimate.
|
||||
// We intentionally preserve the data URL prefix and JSON
|
||||
// wrapper bytes already included in `raw`.
|
||||
raw.saturating_sub(payload_bytes)
|
||||
.saturating_add(image_count.saturating_mul(IMAGE_BYTES_ESTIMATE))
|
||||
.saturating_add(replacement_bytes)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -463,7 +474,7 @@ pub(crate) fn estimate_response_item_model_visible_bytes(item: &ResponseItem) ->
|
||||
///
|
||||
/// We only discount payloads for `data:image/...;base64,...` URLs (case
|
||||
/// insensitive markers) and leave everything else at raw serialized size.
|
||||
fn base64_data_url_payload_len(url: &str) -> Option<usize> {
|
||||
fn parse_base64_image_data_url(url: &str) -> Option<&str> {
|
||||
if !url
|
||||
.get(.."data:".len())
|
||||
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:"))
|
||||
@@ -489,22 +500,67 @@ fn base64_data_url_payload_len(url: &str) -> Option<usize> {
|
||||
if !has_base64_marker {
|
||||
return None;
|
||||
}
|
||||
Some(payload.len())
|
||||
Some(payload)
|
||||
}
|
||||
|
||||
fn base64_data_url_payload_len(url: &str) -> Option<usize> {
|
||||
parse_base64_image_data_url(url).map(str::len)
|
||||
}
|
||||
|
||||
fn estimate_original_image_bytes(image_url: &str) -> Option<i64> {
|
||||
let payload = parse_base64_image_data_url(image_url)?;
|
||||
let bytes = BASE64_STANDARD.decode(payload).ok()?;
|
||||
let dynamic = image::load_from_memory(&bytes).ok()?;
|
||||
let mut width = dynamic.width();
|
||||
let mut height = dynamic.height();
|
||||
|
||||
if width > ORIGINAL_IMAGE_MAX_DIM || height > ORIGINAL_IMAGE_MAX_DIM {
|
||||
let scale = f64::min(
|
||||
f64::from(ORIGINAL_IMAGE_MAX_DIM) / f64::from(width),
|
||||
f64::from(ORIGINAL_IMAGE_MAX_DIM) / f64::from(height),
|
||||
);
|
||||
width = (f64::from(width) * scale).round().max(1.0) as u32;
|
||||
height = (f64::from(height) * scale).round().max(1.0) as u32;
|
||||
}
|
||||
|
||||
let shortest_side = width.min(height);
|
||||
if shortest_side > 0 {
|
||||
let scale = f64::from(ORIGINAL_IMAGE_TARGET_SHORT_SIDE) / f64::from(shortest_side);
|
||||
width = (f64::from(width) * scale).round().max(1.0) as u32;
|
||||
height = (f64::from(height) * scale).round().max(1.0) as u32;
|
||||
}
|
||||
|
||||
let tiles_wide = i64::from(
|
||||
width.saturating_add(ORIGINAL_IMAGE_TILE_SIZE.saturating_sub(1)) / ORIGINAL_IMAGE_TILE_SIZE,
|
||||
);
|
||||
let tiles_high = i64::from(
|
||||
height.saturating_add(ORIGINAL_IMAGE_TILE_SIZE.saturating_sub(1))
|
||||
/ ORIGINAL_IMAGE_TILE_SIZE,
|
||||
);
|
||||
let tile_count = tiles_wide.saturating_mul(tiles_high);
|
||||
let tokens = ORIGINAL_IMAGE_BASE_TOKENS
|
||||
.saturating_add(tile_count.saturating_mul(ORIGINAL_IMAGE_TILE_TOKENS));
|
||||
Some(tokens.saturating_mul(4))
|
||||
}
|
||||
|
||||
/// Scans one response item for discount-eligible inline image data URLs and
|
||||
/// returns:
|
||||
/// - total base64 payload bytes to subtract from raw serialized size
|
||||
/// - count of qualifying images to replace with `IMAGE_BYTES_ESTIMATE`
|
||||
/// - total replacement byte estimate for those images
|
||||
fn image_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) {
|
||||
let mut payload_bytes = 0i64;
|
||||
let mut image_count = 0i64;
|
||||
let mut replacement_bytes = 0i64;
|
||||
|
||||
let mut accumulate = |image_url: &str| {
|
||||
let mut accumulate = |image_url: &str, detail: Option<ImageDetail>| {
|
||||
if let Some(payload_len) = base64_data_url_payload_len(image_url) {
|
||||
payload_bytes =
|
||||
payload_bytes.saturating_add(i64::try_from(payload_len).unwrap_or(i64::MAX));
|
||||
image_count = image_count.saturating_add(1);
|
||||
replacement_bytes = replacement_bytes.saturating_add(match detail {
|
||||
Some(ImageDetail::Original) => {
|
||||
estimate_original_image_bytes(image_url).unwrap_or(RESIZED_IMAGE_BYTES_ESTIMATE)
|
||||
}
|
||||
_ => RESIZED_IMAGE_BYTES_ESTIMATE,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
@@ -512,7 +568,7 @@ fn image_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) {
|
||||
ResponseItem::Message { content, .. } => {
|
||||
for content_item in content {
|
||||
if let ContentItem::InputImage { image_url } = content_item {
|
||||
accumulate(image_url);
|
||||
accumulate(image_url, None);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -520,8 +576,10 @@ fn image_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) {
|
||||
| ResponseItem::CustomToolCallOutput { output, .. } => {
|
||||
if let FunctionCallOutputBody::ContentItems(items) = &output.body {
|
||||
for content_item in items {
|
||||
if let FunctionCallOutputContentItem::InputImage { image_url } = content_item {
|
||||
accumulate(image_url);
|
||||
if let FunctionCallOutputContentItem::InputImage { image_url, detail } =
|
||||
content_item
|
||||
{
|
||||
accumulate(image_url, *detail);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -529,7 +587,7 @@ fn image_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) {
|
||||
_ => {}
|
||||
}
|
||||
|
||||
(payload_bytes, image_count)
|
||||
(payload_bytes, replacement_bytes)
|
||||
}
|
||||
|
||||
fn is_model_generated_item(item: &ResponseItem) -> bool {
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
use super::*;
|
||||
use crate::truncate;
|
||||
use crate::truncate::TruncationPolicy;
|
||||
use base64::Engine;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
|
||||
use codex_git::GhostCommit;
|
||||
use codex_protocol::models::BaseInstructions;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputBody;
|
||||
use codex_protocol::models::FunctionCallOutputContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputPayload;
|
||||
use codex_protocol::models::ImageDetail;
|
||||
use codex_protocol::models::LocalShellAction;
|
||||
use codex_protocol::models::LocalShellExecAction;
|
||||
use codex_protocol::models::LocalShellStatus;
|
||||
@@ -14,6 +17,9 @@ use codex_protocol::models::ReasoningItemContent;
|
||||
use codex_protocol::models::ReasoningItemReasoningSummary;
|
||||
use codex_protocol::openai_models::InputModality;
|
||||
use codex_protocol::openai_models::default_input_modalities;
|
||||
use image::ImageBuffer;
|
||||
use image::ImageFormat;
|
||||
use image::Rgba;
|
||||
use pretty_assertions::assert_eq;
|
||||
use regex_lite::Regex;
|
||||
|
||||
@@ -276,6 +282,7 @@ fn for_prompt_strips_images_when_model_does_not_support_images() {
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "https://example.com/result.png".to_string(),
|
||||
detail: None,
|
||||
},
|
||||
]),
|
||||
},
|
||||
@@ -294,6 +301,7 @@ fn for_prompt_strips_images_when_model_does_not_support_images() {
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "https://example.com/js-repl-result.png".to_string(),
|
||||
detail: None,
|
||||
},
|
||||
]),
|
||||
},
|
||||
@@ -489,6 +497,7 @@ fn replace_last_turn_images_replaces_tool_output_images() {
|
||||
body: FunctionCallOutputBody::ContentItems(vec![
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,AAA".to_string(),
|
||||
detail: None,
|
||||
},
|
||||
]),
|
||||
success: Some(true),
|
||||
@@ -1302,7 +1311,7 @@ fn image_data_url_payload_does_not_dominate_message_estimate() {
|
||||
|
||||
let raw_len = serde_json::to_string(&image_item).unwrap().len() as i64;
|
||||
let estimated = estimate_response_item_model_visible_bytes(&image_item);
|
||||
let expected = raw_len - payload.len() as i64 + IMAGE_BYTES_ESTIMATE;
|
||||
let expected = raw_len - payload.len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;
|
||||
let text_only_estimated = estimate_response_item_model_visible_bytes(&text_only_item);
|
||||
|
||||
assert_eq!(estimated, expected);
|
||||
@@ -1320,13 +1329,16 @@ fn image_data_url_payload_does_not_dominate_function_call_output_estimate() {
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "Screenshot captured".to_string(),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage { image_url },
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url,
|
||||
detail: None,
|
||||
},
|
||||
]),
|
||||
};
|
||||
|
||||
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
|
||||
let estimated = estimate_response_item_model_visible_bytes(&item);
|
||||
let expected = raw_len - payload.len() as i64 + IMAGE_BYTES_ESTIMATE;
|
||||
let expected = raw_len - payload.len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;
|
||||
|
||||
assert_eq!(estimated, expected);
|
||||
assert!(estimated < raw_len);
|
||||
@@ -1342,13 +1354,16 @@ fn image_data_url_payload_does_not_dominate_custom_tool_call_output_estimate() {
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "Screenshot captured".to_string(),
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage { image_url },
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url,
|
||||
detail: None,
|
||||
},
|
||||
]),
|
||||
};
|
||||
|
||||
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
|
||||
let estimated = estimate_response_item_model_visible_bytes(&item);
|
||||
let expected = raw_len - payload.len() as i64 + IMAGE_BYTES_ESTIMATE;
|
||||
let expected = raw_len - payload.len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;
|
||||
|
||||
assert_eq!(estimated, expected);
|
||||
assert!(estimated < raw_len);
|
||||
@@ -1370,6 +1385,7 @@ fn non_base64_image_urls_are_unchanged() {
|
||||
output: FunctionCallOutputPayload::from_content_items(vec![
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "file:///tmp/foo.png".to_string(),
|
||||
detail: None,
|
||||
},
|
||||
]),
|
||||
};
|
||||
@@ -1409,7 +1425,10 @@ fn non_image_base64_data_url_is_unchanged() {
|
||||
let item = ResponseItem::FunctionCallOutput {
|
||||
call_id: "call-octet".to_string(),
|
||||
output: FunctionCallOutputPayload::from_content_items(vec![
|
||||
FunctionCallOutputContentItem::InputImage { image_url },
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url,
|
||||
detail: None,
|
||||
},
|
||||
]),
|
||||
};
|
||||
|
||||
@@ -1433,7 +1452,7 @@ fn mixed_case_data_url_markers_are_adjusted() {
|
||||
|
||||
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
|
||||
let estimated = estimate_response_item_model_visible_bytes(&item);
|
||||
let expected = raw_len - payload.len() as i64 + IMAGE_BYTES_ESTIMATE;
|
||||
let expected = raw_len - payload.len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;
|
||||
|
||||
assert_eq!(estimated, expected);
|
||||
}
|
||||
@@ -1465,7 +1484,52 @@ fn multiple_inline_images_apply_multiple_fixed_costs() {
|
||||
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
|
||||
let payload_sum = (payload_one.len() + payload_two.len()) as i64;
|
||||
let estimated = estimate_response_item_model_visible_bytes(&item);
|
||||
let expected = raw_len - payload_sum + (2 * IMAGE_BYTES_ESTIMATE);
|
||||
let expected = raw_len - payload_sum + (2 * RESIZED_IMAGE_BYTES_ESTIMATE);
|
||||
|
||||
assert_eq!(estimated, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn original_detail_images_scale_with_dimensions() {
|
||||
let width = 2304;
|
||||
let height = 864;
|
||||
let image = ImageBuffer::from_pixel(width, height, Rgba([12u8, 34, 56, 255]));
|
||||
let mut bytes = std::io::Cursor::new(Vec::new());
|
||||
image
|
||||
.write_to(&mut bytes, ImageFormat::Png)
|
||||
.expect("encode png");
|
||||
let payload = BASE64_STANDARD.encode(bytes.get_ref());
|
||||
let image_url = format!("data:image/png;base64,{payload}");
|
||||
let item = ResponseItem::FunctionCallOutput {
|
||||
call_id: "call-original".to_string(),
|
||||
output: FunctionCallOutputPayload::from_content_items(vec![
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url,
|
||||
detail: Some(ImageDetail::Original),
|
||||
},
|
||||
]),
|
||||
};
|
||||
|
||||
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
|
||||
let estimated = estimate_response_item_model_visible_bytes(&item);
|
||||
let scale_to_fit = f64::min(
|
||||
f64::from(ORIGINAL_IMAGE_MAX_DIM) / f64::from(width),
|
||||
f64::from(ORIGINAL_IMAGE_MAX_DIM) / f64::from(height),
|
||||
);
|
||||
let scaled_width = (f64::from(width) * scale_to_fit).round() as u32;
|
||||
let scaled_height = (f64::from(height) * scale_to_fit).round() as u32;
|
||||
let scale_short_side =
|
||||
f64::from(ORIGINAL_IMAGE_TARGET_SHORT_SIDE) / f64::from(scaled_width.min(scaled_height));
|
||||
let tiled_width = (f64::from(scaled_width) * scale_short_side).round() as u32;
|
||||
let tiled_height = (f64::from(scaled_height) * scale_short_side).round() as u32;
|
||||
let tile_count = i64::from(
|
||||
tiled_width.saturating_add(ORIGINAL_IMAGE_TILE_SIZE - 1) / ORIGINAL_IMAGE_TILE_SIZE,
|
||||
) * i64::from(
|
||||
tiled_height.saturating_add(ORIGINAL_IMAGE_TILE_SIZE - 1) / ORIGINAL_IMAGE_TILE_SIZE,
|
||||
);
|
||||
let expected_tokens =
|
||||
ORIGINAL_IMAGE_BASE_TOKENS + tile_count.saturating_mul(ORIGINAL_IMAGE_TILE_TOKENS);
|
||||
let expected = raw_len - payload.len() as i64 + (expected_tokens * 4);
|
||||
|
||||
assert_eq!(estimated, expected);
|
||||
}
|
||||
|
||||
@@ -119,6 +119,8 @@ pub enum Feature {
|
||||
MemoryTool,
|
||||
/// Append additional AGENTS.md guidance to user instructions.
|
||||
ChildAgentsMd,
|
||||
/// Preserve original-resolution bytes for `view_image` tool attachments on supported models.
|
||||
ViewImageOriginalResolution,
|
||||
/// Enforce UTF8 output in Powershell.
|
||||
PowershellUtf8,
|
||||
/// Compress request bodies (zstd) when sending streaming requests to codex-backend.
|
||||
@@ -523,6 +525,12 @@ pub const FEATURES: &[FeatureSpec] = &[
|
||||
stage: Stage::UnderDevelopment,
|
||||
default_enabled: false,
|
||||
},
|
||||
FeatureSpec {
|
||||
id: Feature::ViewImageOriginalResolution,
|
||||
key: "view_image_original_resolution",
|
||||
stage: Stage::UnderDevelopment,
|
||||
default_enabled: false,
|
||||
},
|
||||
FeatureSpec {
|
||||
id: Feature::ApplyPatchFreeform,
|
||||
key: "apply_patch_freeform",
|
||||
|
||||
@@ -55,6 +55,12 @@ fn render_js_repl_instructions(config: &Config) -> Option<String> {
|
||||
section.push_str("- Helpers: `codex.tmpDir` and `codex.tool(name, args?)`.\n");
|
||||
section.push_str("- `codex.tool` executes a normal tool call and resolves to the raw tool output object. Use it for shell and non-shell tools alike.\n");
|
||||
section.push_str("- To share generated images with the model, write a file under `codex.tmpDir`, call `await codex.tool(\"view_image\", { path: \"/absolute/path\" })`, then delete the file.\n");
|
||||
if config
|
||||
.features
|
||||
.enabled(Feature::ViewImageOriginalResolution)
|
||||
{
|
||||
section.push_str("- When generating or converting images for `view_image` in `js_repl`, prefer JPEG at 85% quality unless lossless quality is strictly required; other formats can be used if the user requests them. This keeps uploads smaller and reduces the chance of hitting image size caps.\n");
|
||||
}
|
||||
section.push_str("- Top-level bindings persist across cells. If you hit `SyntaxError: Identifier 'x' has already been declared`, reuse the binding, pick a new name, wrap in `{ ... }` for block scope, or reset the kernel with `js_repl_reset`.\n");
|
||||
section.push_str("- Top-level static import declarations (for example `import x from \"pkg\"`) are currently unsupported in `js_repl`; use dynamic imports with `await import(\"pkg\")` instead.\n");
|
||||
|
||||
@@ -492,6 +498,21 @@ mod tests {
|
||||
assert_eq!(res, expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn js_repl_original_resolution_guidance_is_feature_gated() {
|
||||
let tmp = tempfile::tempdir().expect("tempdir");
|
||||
let mut cfg = make_config(&tmp, 4096, None).await;
|
||||
cfg.features
|
||||
.enable(Feature::JsRepl)
|
||||
.enable(Feature::ViewImageOriginalResolution);
|
||||
|
||||
let res = get_user_instructions(&cfg, None)
|
||||
.await
|
||||
.expect("js_repl instructions expected");
|
||||
let expected = "## JavaScript REPL (Node)\n- Use `js_repl` for Node-backed JavaScript with top-level await in a persistent kernel.\n- `js_repl` is a freeform/custom tool. Direct `js_repl` calls must send raw JavaScript tool input (optionally with first-line `// codex-js-repl: timeout_ms=15000`). Do not wrap code in JSON (for example `{\"code\":\"...\"}`), quotes, or markdown code fences.\n- Helpers: `codex.tmpDir` and `codex.tool(name, args?)`.\n- `codex.tool` executes a normal tool call and resolves to the raw tool output object. Use it for shell and non-shell tools alike.\n- To share generated images with the model, write a file under `codex.tmpDir`, call `await codex.tool(\"view_image\", { path: \"/absolute/path\" })`, then delete the file.\n- When generating or converting images for `view_image` in `js_repl`, prefer JPEG at 85% quality unless lossless quality is strictly required; other formats can be used if the user requests them. This keeps uploads smaller and reduces the chance of hitting image size caps.\n- Top-level bindings persist across cells. If you hit `SyntaxError: Identifier 'x' has already been declared`, reuse the binding, pick a new name, wrap in `{ ... }` for block scope, or reset the kernel with `js_repl_reset`.\n- Top-level static import declarations (for example `import x from \"pkg\"`) are currently unsupported in `js_repl`; use dynamic imports with `await import(\"pkg\")` instead.\n- Avoid direct access to `process.stdout` / `process.stderr` / `process.stdin`; it can corrupt the JSON line protocol. Use `console.log` and `codex.tool(...)`.";
|
||||
assert_eq!(res, expected);
|
||||
}
|
||||
|
||||
/// When both system instructions *and* a project doc are present the two
|
||||
/// should be concatenated with the separator.
|
||||
#[tokio::test]
|
||||
|
||||
@@ -222,6 +222,7 @@ mod tests {
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,AAA".to_string(),
|
||||
detail: None,
|
||||
},
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "line 2".to_string(),
|
||||
@@ -239,6 +240,7 @@ mod tests {
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,AAA".to_string(),
|
||||
detail: None,
|
||||
},
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "line 2".to_string(),
|
||||
|
||||
@@ -1,10 +1,15 @@
|
||||
use async_trait::async_trait;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputBody;
|
||||
use codex_protocol::models::FunctionCallOutputContentItem;
|
||||
use codex_protocol::models::ImageDetail;
|
||||
use codex_protocol::models::local_image_content_items_with_label_number;
|
||||
use codex_protocol::openai_models::InputModality;
|
||||
use codex_utils_image::PromptImageMode;
|
||||
use serde::Deserialize;
|
||||
use tokio::fs;
|
||||
|
||||
use crate::features::Feature;
|
||||
use crate::function_tool::FunctionCallError;
|
||||
use crate::protocol::EventMsg;
|
||||
use crate::protocol::ViewImageToolCallEvent;
|
||||
@@ -14,19 +19,44 @@ use crate::tools::context::ToolPayload;
|
||||
use crate::tools::handlers::parse_arguments;
|
||||
use crate::tools::registry::ToolHandler;
|
||||
use crate::tools::registry::ToolKind;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::local_image_content_items_with_label_number;
|
||||
|
||||
pub struct ViewImageHandler;
|
||||
|
||||
const VIEW_IMAGE_UNSUPPORTED_MESSAGE: &str =
|
||||
"view_image is not allowed because you do not support image inputs";
|
||||
const MIN_ORIGINAL_RESOLUTION_MODEL_VERSION: (u32, u32) = (5, 4);
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ViewImageArgs {
|
||||
path: String,
|
||||
}
|
||||
|
||||
fn supports_original_resolution_model(slug: &str) -> bool {
|
||||
// Match `gpt-X.Y...` model slugs and enable original-resolution images for
|
||||
// GPT models at version 5.4 or newer.
|
||||
let Some(version_suffix) = slug.strip_prefix("gpt-") else {
|
||||
return false;
|
||||
};
|
||||
let version_end = version_suffix
|
||||
.find(|ch: char| !ch.is_ascii_digit() && ch != '.')
|
||||
.unwrap_or(version_suffix.len());
|
||||
let version = &version_suffix[..version_end];
|
||||
if version.is_empty() {
|
||||
return false;
|
||||
}
|
||||
|
||||
let mut parts = version.split('.');
|
||||
let Some(major) = parts.next().and_then(|part| part.parse::<u32>().ok()) else {
|
||||
return false;
|
||||
};
|
||||
let minor = parts
|
||||
.next()
|
||||
.and_then(|part| part.parse::<u32>().ok())
|
||||
.unwrap_or(0);
|
||||
|
||||
(major, minor) >= MIN_ORIGINAL_RESOLUTION_MODEL_VERSION
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ToolHandler for ViewImageHandler {
|
||||
fn kind(&self) -> ToolKind {
|
||||
@@ -81,15 +111,33 @@ impl ToolHandler for ViewImageHandler {
|
||||
}
|
||||
let event_path = abs_path.clone();
|
||||
|
||||
let content = local_image_content_items_with_label_number(&abs_path, None);
|
||||
let content = content
|
||||
let use_original_resolution = turn
|
||||
.config
|
||||
.features
|
||||
.enabled(Feature::ViewImageOriginalResolution)
|
||||
&& supports_original_resolution_model(&turn.model_info.slug);
|
||||
let image_mode = if use_original_resolution {
|
||||
PromptImageMode::Original
|
||||
} else {
|
||||
PromptImageMode::ResizeToFit
|
||||
};
|
||||
let image_detail = if use_original_resolution {
|
||||
Some(ImageDetail::Original)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let content = local_image_content_items_with_label_number(&abs_path, None, image_mode)
|
||||
.into_iter()
|
||||
.map(|item| match item {
|
||||
ContentItem::InputText { text } => {
|
||||
FunctionCallOutputContentItem::InputText { text }
|
||||
}
|
||||
ContentItem::InputImage { image_url } => {
|
||||
FunctionCallOutputContentItem::InputImage { image_url }
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url,
|
||||
detail: image_detail,
|
||||
}
|
||||
}
|
||||
ContentItem::OutputText { text } => {
|
||||
FunctionCallOutputContentItem::InputText { text }
|
||||
@@ -113,3 +161,21 @@ impl ToolHandler for ViewImageHandler {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::supports_original_resolution_model;
|
||||
|
||||
#[test]
|
||||
fn supports_original_resolution_model_version_gate() {
|
||||
assert!(!supports_original_resolution_model("gpt-4.1"));
|
||||
assert!(!supports_original_resolution_model("gpt-5"));
|
||||
assert!(!supports_original_resolution_model("gpt-5.3"));
|
||||
assert!(supports_original_resolution_model("gpt-5.4"));
|
||||
assert!(supports_original_resolution_model("gpt-5.4-mini"));
|
||||
assert!(supports_original_resolution_model("gpt-5.5"));
|
||||
assert!(supports_original_resolution_model("gpt-5.10"));
|
||||
assert!(supports_original_resolution_model("gpt-6"));
|
||||
assert!(!supports_original_resolution_model("o3"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1900,6 +1900,7 @@ mod tests {
|
||||
output: FunctionCallOutputPayload::from_content_items(vec![
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,abcd".to_string(),
|
||||
detail: None,
|
||||
},
|
||||
]),
|
||||
};
|
||||
@@ -1929,6 +1930,7 @@ mod tests {
|
||||
output: FunctionCallOutputPayload::from_content_items(vec![
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,abcd".to_string(),
|
||||
detail: None,
|
||||
},
|
||||
]),
|
||||
};
|
||||
@@ -2417,15 +2419,17 @@ console.log(out.output?.body?.text ?? "");
|
||||
image_url:
|
||||
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg=="
|
||||
.to_string(),
|
||||
detail: None,
|
||||
}]
|
||||
.as_slice()
|
||||
);
|
||||
let [FunctionCallOutputContentItem::InputImage { image_url }] =
|
||||
let [FunctionCallOutputContentItem::InputImage { image_url, detail }] =
|
||||
result.content_items.as_slice()
|
||||
else {
|
||||
panic!("view_image should return exactly one input_image content item");
|
||||
};
|
||||
assert!(image_url.starts_with("data:image/png;base64,"));
|
||||
assert_eq!(*detail, None);
|
||||
assert!(session.get_pending_input().await.is_empty());
|
||||
|
||||
Ok(())
|
||||
@@ -2515,6 +2519,7 @@ console.log(out.type);
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: image_url.to_string(),
|
||||
detail: None,
|
||||
},
|
||||
]
|
||||
);
|
||||
|
||||
@@ -138,9 +138,10 @@ pub(crate) fn truncate_function_output_items_with_policy(
|
||||
remaining_budget = 0;
|
||||
}
|
||||
}
|
||||
FunctionCallOutputContentItem::InputImage { image_url } => {
|
||||
FunctionCallOutputContentItem::InputImage { image_url, detail } => {
|
||||
out.push(FunctionCallOutputContentItem::InputImage {
|
||||
image_url: image_url.clone(),
|
||||
detail: *detail,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -491,6 +492,7 @@ mod tests {
|
||||
FunctionCallOutputContentItem::InputText { text: t2.clone() },
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "img:mid".to_string(),
|
||||
detail: None,
|
||||
},
|
||||
FunctionCallOutputContentItem::InputText { text: t3 },
|
||||
FunctionCallOutputContentItem::InputText { text: t4 },
|
||||
@@ -518,7 +520,8 @@ mod tests {
|
||||
assert_eq!(
|
||||
output[2],
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "img:mid".to_string()
|
||||
image_url: "img:mid".to_string(),
|
||||
detail: None,
|
||||
}
|
||||
);
|
||||
|
||||
|
||||
@@ -289,6 +289,204 @@ async fn view_image_tool_attaches_local_image() -> anyhow::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn view_image_tool_can_preserve_original_resolution_on_gpt5_4() -> anyhow::Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
let server = start_mock_server().await;
|
||||
let mut builder = test_codex().with_model("gpt-5.4").with_config(|config| {
|
||||
config.features.enable(Feature::ViewImageOriginalResolution);
|
||||
});
|
||||
let TestCodex {
|
||||
codex,
|
||||
cwd,
|
||||
session_configured,
|
||||
..
|
||||
} = builder.build(&server).await?;
|
||||
|
||||
let rel_path = "assets/original-example.png";
|
||||
let abs_path = cwd.path().join(rel_path);
|
||||
if let Some(parent) = abs_path.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
let original_width = 2304;
|
||||
let original_height = 864;
|
||||
let image = ImageBuffer::from_pixel(original_width, original_height, Rgba([0u8, 80, 255, 255]));
|
||||
image.save(&abs_path)?;
|
||||
|
||||
let call_id = "view-image-original";
|
||||
let arguments = serde_json::json!({ "path": rel_path }).to_string();
|
||||
|
||||
let first_response = sse(vec![
|
||||
ev_response_created("resp-1"),
|
||||
ev_function_call(call_id, "view_image", &arguments),
|
||||
ev_completed("resp-1"),
|
||||
]);
|
||||
responses::mount_sse_once(&server, first_response).await;
|
||||
|
||||
let second_response = sse(vec![
|
||||
ev_assistant_message("msg-1", "done"),
|
||||
ev_completed("resp-2"),
|
||||
]);
|
||||
let mock = responses::mount_sse_once(&server, second_response).await;
|
||||
|
||||
let session_model = session_configured.model.clone();
|
||||
|
||||
codex
|
||||
.submit(Op::UserTurn {
|
||||
items: vec![UserInput::Text {
|
||||
text: "please add the original screenshot".into(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
final_output_json_schema: None,
|
||||
cwd: cwd.path().to_path_buf(),
|
||||
approval_policy: AskForApproval::Never,
|
||||
sandbox_policy: SandboxPolicy::DangerFullAccess,
|
||||
model: session_model,
|
||||
effort: None,
|
||||
summary: None,
|
||||
collaboration_mode: None,
|
||||
personality: None,
|
||||
})
|
||||
.await?;
|
||||
|
||||
wait_for_event_with_timeout(
|
||||
&codex,
|
||||
|event| matches!(event, EventMsg::TurnComplete(_)),
|
||||
Duration::from_secs(10),
|
||||
)
|
||||
.await;
|
||||
|
||||
let req = mock.single_request();
|
||||
let function_output = req.function_call_output(call_id);
|
||||
let output_items = function_output
|
||||
.get("output")
|
||||
.and_then(Value::as_array)
|
||||
.expect("function_call_output should be a content item array");
|
||||
assert_eq!(output_items.len(), 1);
|
||||
assert_eq!(
|
||||
output_items[0].get("detail").and_then(Value::as_str),
|
||||
Some("original")
|
||||
);
|
||||
let image_url = output_items[0]
|
||||
.get("image_url")
|
||||
.and_then(Value::as_str)
|
||||
.expect("image_url present");
|
||||
|
||||
let (_, encoded) = image_url
|
||||
.split_once(',')
|
||||
.expect("image url contains data prefix");
|
||||
let decoded = BASE64_STANDARD
|
||||
.decode(encoded)
|
||||
.expect("image data decodes from base64 for request");
|
||||
let preserved = load_from_memory(&decoded).expect("load preserved image");
|
||||
let (width, height) = preserved.dimensions();
|
||||
assert_eq!(width, original_width);
|
||||
assert_eq!(height, original_height);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn view_image_tool_keeps_legacy_behavior_below_gpt5_4() -> anyhow::Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
let server = start_mock_server().await;
|
||||
let mut builder = test_codex().with_model("gpt-5.3").with_config(|config| {
|
||||
config.features.enable(Feature::ViewImageOriginalResolution);
|
||||
});
|
||||
let TestCodex {
|
||||
codex,
|
||||
cwd,
|
||||
session_configured,
|
||||
..
|
||||
} = builder.build(&server).await?;
|
||||
|
||||
let rel_path = "assets/original-example-lower-model.png";
|
||||
let abs_path = cwd.path().join(rel_path);
|
||||
if let Some(parent) = abs_path.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
let original_width = 2304;
|
||||
let original_height = 864;
|
||||
let image = ImageBuffer::from_pixel(original_width, original_height, Rgba([0u8, 80, 255, 255]));
|
||||
image.save(&abs_path)?;
|
||||
|
||||
let call_id = "view-image-original-lower-model";
|
||||
let arguments = serde_json::json!({ "path": rel_path }).to_string();
|
||||
|
||||
let first_response = sse(vec![
|
||||
ev_response_created("resp-1"),
|
||||
ev_function_call(call_id, "view_image", &arguments),
|
||||
ev_completed("resp-1"),
|
||||
]);
|
||||
responses::mount_sse_once(&server, first_response).await;
|
||||
|
||||
let second_response = sse(vec![
|
||||
ev_assistant_message("msg-1", "done"),
|
||||
ev_completed("resp-2"),
|
||||
]);
|
||||
let mock = responses::mount_sse_once(&server, second_response).await;
|
||||
|
||||
let session_model = session_configured.model.clone();
|
||||
|
||||
codex
|
||||
.submit(Op::UserTurn {
|
||||
items: vec![UserInput::Text {
|
||||
text: "please add the screenshot".into(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
final_output_json_schema: None,
|
||||
cwd: cwd.path().to_path_buf(),
|
||||
approval_policy: AskForApproval::Never,
|
||||
sandbox_policy: SandboxPolicy::DangerFullAccess,
|
||||
model: session_model,
|
||||
effort: None,
|
||||
summary: None,
|
||||
collaboration_mode: None,
|
||||
personality: None,
|
||||
})
|
||||
.await?;
|
||||
|
||||
wait_for_event_with_timeout(
|
||||
&codex,
|
||||
|event| matches!(event, EventMsg::TurnComplete(_)),
|
||||
Duration::from_secs(10),
|
||||
)
|
||||
.await;
|
||||
|
||||
let req = mock.single_request();
|
||||
let function_output = req.function_call_output(call_id);
|
||||
let output_items = function_output
|
||||
.get("output")
|
||||
.and_then(Value::as_array)
|
||||
.expect("function_call_output should be a content item array");
|
||||
assert_eq!(output_items.len(), 1);
|
||||
assert_eq!(output_items[0].get("detail"), None);
|
||||
|
||||
let image_url = output_items[0]
|
||||
.get("image_url")
|
||||
.and_then(Value::as_str)
|
||||
.expect("image_url present");
|
||||
|
||||
let (prefix, encoded) = image_url
|
||||
.split_once(',')
|
||||
.expect("image url contains data prefix");
|
||||
assert_eq!(prefix, "data:image/png;base64");
|
||||
|
||||
let decoded = BASE64_STANDARD
|
||||
.decode(encoded)
|
||||
.expect("image data decodes from base64 for request");
|
||||
let resized = load_from_memory(&decoded).expect("load resized image");
|
||||
let (resized_width, resized_height) = resized.dimensions();
|
||||
assert!(resized_width <= 2048);
|
||||
assert!(resized_height <= 768);
|
||||
assert!(resized_width < original_width);
|
||||
assert!(resized_height < original_height);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn js_repl_view_image_tool_attaches_local_image() -> anyhow::Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
|
||||
use codex_utils_image::load_and_resize_to_fit;
|
||||
use codex_utils_image::PromptImageMode;
|
||||
use codex_utils_image::load_for_prompt;
|
||||
use serde::Deserialize;
|
||||
use serde::Deserializer;
|
||||
use serde::Serialize;
|
||||
@@ -173,6 +174,15 @@ pub enum ContentItem {
|
||||
OutputText { text: String },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, JsonSchema, TS)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum ImageDetail {
|
||||
Auto,
|
||||
Low,
|
||||
High,
|
||||
Original,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema, TS)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
/// Classifies an assistant message as interim commentary or final answer text.
|
||||
@@ -691,8 +701,9 @@ fn unsupported_image_error_placeholder(path: &std::path::Path, mime: &str) -> Co
|
||||
pub fn local_image_content_items_with_label_number(
|
||||
path: &std::path::Path,
|
||||
label_number: Option<usize>,
|
||||
mode: PromptImageMode,
|
||||
) -> Vec<ContentItem> {
|
||||
match load_and_resize_to_fit(path) {
|
||||
match load_for_prompt(path, mode) {
|
||||
Ok(image) => {
|
||||
let mut items = Vec::with_capacity(3);
|
||||
if let Some(label_number) = label_number {
|
||||
@@ -853,7 +864,11 @@ impl From<Vec<UserInput>> for ResponseInputItem {
|
||||
}
|
||||
UserInput::LocalImage { path } => {
|
||||
image_index += 1;
|
||||
local_image_content_items_with_label_number(&path, Some(image_index))
|
||||
local_image_content_items_with_label_number(
|
||||
&path,
|
||||
Some(image_index),
|
||||
PromptImageMode::ResizeToFit,
|
||||
)
|
||||
}
|
||||
UserInput::Skill { .. } | UserInput::Mention { .. } => Vec::new(), // Tool bodies are injected later in core
|
||||
})
|
||||
@@ -918,9 +933,16 @@ pub struct ShellCommandToolCallParams {
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum FunctionCallOutputContentItem {
|
||||
// Do not rename, these are serialized and used directly in the responses API.
|
||||
InputText { text: String },
|
||||
InputText {
|
||||
text: String,
|
||||
},
|
||||
// Do not rename, these are serialized and used directly in the responses API.
|
||||
InputImage { image_url: String },
|
||||
InputImage {
|
||||
image_url: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[ts(optional)]
|
||||
detail: Option<ImageDetail>,
|
||||
},
|
||||
}
|
||||
|
||||
/// Converts structured function-call output content into plain text for
|
||||
@@ -964,7 +986,10 @@ impl From<crate::dynamic_tools::DynamicToolCallOutputContentItem>
|
||||
Self::InputText { text }
|
||||
}
|
||||
crate::dynamic_tools::DynamicToolCallOutputContentItem::InputImage { image_url } => {
|
||||
Self::InputImage { image_url }
|
||||
Self::InputImage {
|
||||
image_url,
|
||||
detail: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1166,7 +1191,10 @@ fn convert_mcp_content_to_items(
|
||||
let mime_type = mime_type.unwrap_or_else(|| "application/octet-stream".into());
|
||||
format!("data:{mime_type};base64,{data}")
|
||||
};
|
||||
FunctionCallOutputContentItem::InputImage { image_url }
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url,
|
||||
detail: None,
|
||||
}
|
||||
}
|
||||
Ok(McpContent::Unknown) | Err(_) => FunctionCallOutputContentItem::InputText {
|
||||
text: serde_json::to_string(content).unwrap_or_else(|_| "<content>".to_string()),
|
||||
@@ -1220,6 +1248,7 @@ mod tests {
|
||||
items,
|
||||
vec![FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,Zm9v".to_string(),
|
||||
detail: None,
|
||||
}]
|
||||
);
|
||||
}
|
||||
@@ -1237,6 +1266,7 @@ mod tests {
|
||||
items,
|
||||
vec![FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,Zm9v".to_string(),
|
||||
detail: None,
|
||||
}]
|
||||
);
|
||||
}
|
||||
@@ -1259,6 +1289,7 @@ mod tests {
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,AAA".to_string(),
|
||||
detail: None,
|
||||
},
|
||||
FunctionCallOutputContentItem::InputText {
|
||||
text: "line 2".to_string(),
|
||||
@@ -1277,6 +1308,7 @@ mod tests {
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,AAA".to_string(),
|
||||
detail: None,
|
||||
},
|
||||
];
|
||||
|
||||
@@ -1299,6 +1331,7 @@ mod tests {
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,AAA".to_string(),
|
||||
detail: None,
|
||||
},
|
||||
]);
|
||||
|
||||
@@ -1523,6 +1556,7 @@ mod tests {
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,BASE64".into(),
|
||||
detail: None,
|
||||
},
|
||||
]
|
||||
);
|
||||
@@ -1548,6 +1582,7 @@ mod tests {
|
||||
output: FunctionCallOutputPayload::from_content_items(vec![
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,BASE64".into(),
|
||||
detail: None,
|
||||
},
|
||||
]),
|
||||
};
|
||||
@@ -1583,6 +1618,7 @@ mod tests {
|
||||
items,
|
||||
vec![FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,BASE64".into(),
|
||||
detail: None,
|
||||
}]
|
||||
);
|
||||
|
||||
@@ -1605,6 +1641,7 @@ mod tests {
|
||||
},
|
||||
FunctionCallOutputContentItem::InputImage {
|
||||
image_url: "data:image/png;base64,XYZ".into(),
|
||||
detail: None,
|
||||
},
|
||||
];
|
||||
assert_eq!(
|
||||
|
||||
@@ -33,24 +33,48 @@ pub struct EncodedImage {
|
||||
impl EncodedImage {
|
||||
pub fn into_data_url(self) -> String {
|
||||
let encoded = BASE64_STANDARD.encode(&self.bytes);
|
||||
format!("data:{};base64,{}", self.mime, encoded)
|
||||
format!("data:{};base64,{encoded}", self.mime)
|
||||
}
|
||||
}
|
||||
|
||||
static IMAGE_CACHE: LazyLock<BlockingLruCache<[u8; 20], EncodedImage>> =
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum PromptImageMode {
|
||||
ResizeToFit,
|
||||
Original,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
struct ImageCacheKey {
|
||||
digest: [u8; 20],
|
||||
mode: PromptImageMode,
|
||||
}
|
||||
|
||||
static IMAGE_CACHE: LazyLock<BlockingLruCache<ImageCacheKey, EncodedImage>> =
|
||||
LazyLock::new(|| BlockingLruCache::new(NonZeroUsize::new(32).unwrap_or(NonZeroUsize::MIN)));
|
||||
|
||||
pub fn load_and_resize_to_fit(path: &Path) -> Result<EncodedImage, ImageProcessingError> {
|
||||
load_for_prompt(path, PromptImageMode::ResizeToFit)
|
||||
}
|
||||
|
||||
pub fn load_for_prompt(
|
||||
path: &Path,
|
||||
mode: PromptImageMode,
|
||||
) -> Result<EncodedImage, ImageProcessingError> {
|
||||
let path_buf = path.to_path_buf();
|
||||
|
||||
let file_bytes = read_file_bytes(path, &path_buf)?;
|
||||
|
||||
let key = sha1_digest(&file_bytes);
|
||||
let key = ImageCacheKey {
|
||||
digest: sha1_digest(&file_bytes),
|
||||
mode,
|
||||
};
|
||||
|
||||
IMAGE_CACHE.get_or_try_insert_with(key, move || {
|
||||
let format = match image::guess_format(&file_bytes) {
|
||||
Ok(ImageFormat::Png) => Some(ImageFormat::Png),
|
||||
Ok(ImageFormat::Jpeg) => Some(ImageFormat::Jpeg),
|
||||
Ok(ImageFormat::Gif) => Some(ImageFormat::Gif),
|
||||
Ok(ImageFormat::WebP) => Some(ImageFormat::WebP),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
@@ -63,8 +87,10 @@ pub fn load_and_resize_to_fit(path: &Path) -> Result<EncodedImage, ImageProcessi
|
||||
|
||||
let (width, height) = dynamic.dimensions();
|
||||
|
||||
let encoded = if width <= MAX_WIDTH && height <= MAX_HEIGHT {
|
||||
if let Some(format) = format {
|
||||
let encoded = if mode == PromptImageMode::Original
|
||||
|| (width <= MAX_WIDTH && height <= MAX_HEIGHT)
|
||||
{
|
||||
if let Some(format) = format.filter(|format| can_preserve_source_bytes(*format, mode)) {
|
||||
let mime = format_to_mime(format);
|
||||
EncodedImage {
|
||||
bytes: file_bytes,
|
||||
@@ -99,6 +125,20 @@ pub fn load_and_resize_to_fit(path: &Path) -> Result<EncodedImage, ImageProcessi
|
||||
})
|
||||
}
|
||||
|
||||
fn can_preserve_source_bytes(format: ImageFormat, mode: PromptImageMode) -> bool {
|
||||
match mode {
|
||||
PromptImageMode::ResizeToFit => matches!(format, ImageFormat::Png | ImageFormat::Jpeg),
|
||||
PromptImageMode::Original => {
|
||||
// Public API docs explicitly call out non-animated GIF support only.
|
||||
// Preserve byte-for-byte only for formats we can safely pass through.
|
||||
matches!(
|
||||
format,
|
||||
ImageFormat::Png | ImageFormat::Jpeg | ImageFormat::WebP
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn read_file_bytes(path: &Path, path_for_error: &Path) -> Result<Vec<u8>, ImageProcessingError> {
|
||||
match tokio::runtime::Handle::try_current() {
|
||||
// If we're inside a Tokio runtime, avoid block_on (it panics on worker threads).
|
||||
@@ -162,6 +202,8 @@ fn encode_image(
|
||||
fn format_to_mime(format: ImageFormat) -> String {
|
||||
match format {
|
||||
ImageFormat::Jpeg => "image/jpeg".to_string(),
|
||||
ImageFormat::Gif => "image/gif".to_string(),
|
||||
ImageFormat::WebP => "image/webp".to_string(),
|
||||
_ => "image/png".to_string(),
|
||||
}
|
||||
}
|
||||
@@ -210,6 +252,24 @@ mod tests {
|
||||
assert_eq!(loaded.dimensions(), (processed.width, processed.height));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn preserves_large_image_in_original_mode() {
|
||||
let temp_file = NamedTempFile::new().expect("temp file");
|
||||
let image = ImageBuffer::from_pixel(4096, 2048, Rgba([180u8, 30, 30, 255]));
|
||||
image
|
||||
.save_with_format(temp_file.path(), ImageFormat::Png)
|
||||
.expect("write png to temp file");
|
||||
|
||||
let original_bytes = std::fs::read(temp_file.path()).expect("read written image");
|
||||
let processed =
|
||||
load_for_prompt(temp_file.path(), PromptImageMode::Original).expect("process image");
|
||||
|
||||
assert_eq!(processed.width, 4096);
|
||||
assert_eq!(processed.height, 2048);
|
||||
assert_eq!(processed.mime, "image/png");
|
||||
assert_eq!(processed.bytes, original_bytes);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn fails_cleanly_for_invalid_images() {
|
||||
let temp_file = NamedTempFile::new().expect("temp file");
|
||||
|
||||
Reference in New Issue
Block a user