Move string truncation helpers into codex-utils-string (#15572)

- move the shared byte-based middle truncation logic from `core` into
`codex-utils-string`
- keep token-specific truncation in `codex-core` so rollout can reuse
the shared helper in the next stacked PR

---------

Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Ahmed Ibrahim
2026-03-24 15:45:40 -07:00
committed by GitHub
parent 0b619afc87
commit 062fa7a2bb
36 changed files with 551 additions and 487 deletions

View File

@@ -7,6 +7,7 @@ use std::collections::HashMap;
use std::collections::HashSet;
use std::ffi::OsStr;
use std::fmt;
use std::ops::Mul;
use std::path::Path;
use std::path::PathBuf;
use std::str::FromStr;
@@ -2629,6 +2630,51 @@ pub enum TruncationPolicy {
Tokens(usize),
}
impl From<crate::openai_models::TruncationPolicyConfig> for TruncationPolicy {
fn from(config: crate::openai_models::TruncationPolicyConfig) -> Self {
match config.mode {
crate::openai_models::TruncationMode::Bytes => Self::Bytes(config.limit as usize),
crate::openai_models::TruncationMode::Tokens => Self::Tokens(config.limit as usize),
}
}
}
impl TruncationPolicy {
pub fn token_budget(&self) -> usize {
match self {
TruncationPolicy::Bytes(bytes) => {
usize::try_from(codex_utils_string::approx_tokens_from_byte_count(*bytes))
.unwrap_or(usize::MAX)
}
TruncationPolicy::Tokens(tokens) => *tokens,
}
}
pub fn byte_budget(&self) -> usize {
match self {
TruncationPolicy::Bytes(bytes) => *bytes,
TruncationPolicy::Tokens(tokens) => {
codex_utils_string::approx_bytes_for_tokens(*tokens)
}
}
}
}
impl Mul<f64> for TruncationPolicy {
type Output = Self;
fn mul(self, multiplier: f64) -> Self::Output {
match self {
TruncationPolicy::Bytes(bytes) => {
TruncationPolicy::Bytes((bytes as f64 * multiplier).ceil() as usize)
}
TruncationPolicy::Tokens(tokens) => {
TruncationPolicy::Tokens((tokens as f64 * multiplier).ceil() as usize)
}
}
}
}
#[derive(Serialize, Deserialize, Clone, JsonSchema)]
pub struct RolloutLine {
pub timestamp: String,