chore: add web_search_tool_type for image support (#13538)

add `web_search_tool_type` on model_info that can be populated from
backend. will be used to filter which models can use `web_search` with
images and which cant.

added small unit test.
This commit is contained in:
sayan-oai
2026-03-04 23:02:27 -08:00
committed by GitHub
parent 8f828f8a43
commit 03d55f0e6f
12 changed files with 89 additions and 12 deletions

View File

@@ -25,6 +25,7 @@ use codex_protocol::openai_models::ApplyPatchToolType;
use codex_protocol::openai_models::ConfigShellToolType;
use codex_protocol::openai_models::InputModality;
use codex_protocol::openai_models::ModelInfo;
use codex_protocol::openai_models::WebSearchToolType;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::SubAgentSource;
use serde::Deserialize;
@@ -36,6 +37,7 @@ use std::collections::HashMap;
const SEARCH_TOOL_BM25_DESCRIPTION_TEMPLATE: &str =
include_str!("../../templates/search_tool/tool_description.md");
const WEB_SEARCH_CONTENT_TYPES: [&str; 2] = ["text", "image"];
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
pub enum ShellCommandBackendConfig {
Classic,
@@ -49,6 +51,7 @@ pub(crate) struct ToolsConfig {
pub allow_login_shell: bool,
pub apply_patch_tool_type: Option<ApplyPatchToolType>,
pub web_search_mode: Option<WebSearchMode>,
pub web_search_tool_type: WebSearchToolType,
pub image_gen_tool: bool,
pub agent_roles: BTreeMap<String, AgentRoleConfig>,
pub search_tool: bool,
@@ -140,6 +143,7 @@ impl ToolsConfig {
allow_login_shell: true,
apply_patch_tool_type,
web_search_mode: *web_search_mode,
web_search_tool_type: model_info.web_search_tool_type,
image_gen_tool: include_image_gen_tool,
agent_roles: BTreeMap::new(),
search_tool: include_search_tool,
@@ -1877,18 +1881,27 @@ pub(crate) fn build_specs(
builder.register_handler("test_sync_tool", test_sync_handler);
}
match config.web_search_mode {
Some(WebSearchMode::Cached) => {
builder.push_spec(ToolSpec::WebSearch {
external_web_access: Some(false),
});
}
Some(WebSearchMode::Live) => {
builder.push_spec(ToolSpec::WebSearch {
external_web_access: Some(true),
});
}
Some(WebSearchMode::Disabled) | None => {}
let external_web_access = match config.web_search_mode {
Some(WebSearchMode::Cached) => Some(false),
Some(WebSearchMode::Live) => Some(true),
Some(WebSearchMode::Disabled) | None => None,
};
if let Some(external_web_access) = external_web_access {
let search_content_types = match config.web_search_tool_type {
WebSearchToolType::Text => None,
WebSearchToolType::TextAndImage => Some(
WEB_SEARCH_CONTENT_TYPES
.into_iter()
.map(str::to_string)
.collect(),
),
};
builder.push_spec(ToolSpec::WebSearch {
external_web_access: Some(external_web_access),
search_content_types,
});
}
if config.image_gen_tool {
@@ -2172,6 +2185,7 @@ mod tests {
create_apply_patch_freeform_tool(),
ToolSpec::WebSearch {
external_web_access: Some(true),
search_content_types: None,
},
create_view_image_tool(),
] {
@@ -2486,6 +2500,7 @@ mod tests {
tool.spec,
ToolSpec::WebSearch {
external_web_access: Some(false),
search_content_types: None,
}
);
}
@@ -2510,6 +2525,38 @@ mod tests {
tool.spec,
ToolSpec::WebSearch {
external_web_access: Some(true),
search_content_types: None,
}
);
}
#[test]
fn web_search_tool_type_text_and_image_sets_search_content_types() {
let config = test_config();
let mut model_info =
ModelsManager::construct_model_info_offline_for_tests("gpt-5-codex", &config);
model_info.web_search_tool_type = WebSearchToolType::TextAndImage;
let features = Features::with_defaults();
let tools_config = ToolsConfig::new(&ToolsConfigParams {
model_info: &model_info,
features: &features,
web_search_mode: Some(WebSearchMode::Live),
session_source: SessionSource::Cli,
});
let (tools, _) = build_specs(&tools_config, None, None, &[]).build();
let tool = find_tool(&tools, "web_search");
assert_eq!(
tool.spec,
ToolSpec::WebSearch {
external_web_access: Some(true),
search_content_types: Some(
WEB_SEARCH_CONTENT_TYPES
.into_iter()
.map(str::to_string)
.collect()
),
}
);
}