mirror of
https://github.com/openai/codex.git
synced 2026-04-30 03:12:20 +03:00
Move string truncation helpers into codex-utils-string (#15572)
- move the shared byte-based middle truncation logic from `core` into `codex-utils-string` - keep token-specific truncation in `codex-core` so rollout can reuse the shared helper in the next stacked PR --------- Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -7,6 +7,7 @@ use std::collections::HashMap;
|
||||
use std::collections::HashSet;
|
||||
use std::ffi::OsStr;
|
||||
use std::fmt;
|
||||
use std::ops::Mul;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
use std::str::FromStr;
|
||||
@@ -2629,6 +2630,51 @@ pub enum TruncationPolicy {
|
||||
Tokens(usize),
|
||||
}
|
||||
|
||||
impl From<crate::openai_models::TruncationPolicyConfig> for TruncationPolicy {
|
||||
fn from(config: crate::openai_models::TruncationPolicyConfig) -> Self {
|
||||
match config.mode {
|
||||
crate::openai_models::TruncationMode::Bytes => Self::Bytes(config.limit as usize),
|
||||
crate::openai_models::TruncationMode::Tokens => Self::Tokens(config.limit as usize),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TruncationPolicy {
|
||||
pub fn token_budget(&self) -> usize {
|
||||
match self {
|
||||
TruncationPolicy::Bytes(bytes) => {
|
||||
usize::try_from(codex_utils_string::approx_tokens_from_byte_count(*bytes))
|
||||
.unwrap_or(usize::MAX)
|
||||
}
|
||||
TruncationPolicy::Tokens(tokens) => *tokens,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn byte_budget(&self) -> usize {
|
||||
match self {
|
||||
TruncationPolicy::Bytes(bytes) => *bytes,
|
||||
TruncationPolicy::Tokens(tokens) => {
|
||||
codex_utils_string::approx_bytes_for_tokens(*tokens)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Mul<f64> for TruncationPolicy {
|
||||
type Output = Self;
|
||||
|
||||
fn mul(self, multiplier: f64) -> Self::Output {
|
||||
match self {
|
||||
TruncationPolicy::Bytes(bytes) => {
|
||||
TruncationPolicy::Bytes((bytes as f64 * multiplier).ceil() as usize)
|
||||
}
|
||||
TruncationPolicy::Tokens(tokens) => {
|
||||
TruncationPolicy::Tokens((tokens as f64 * multiplier).ceil() as usize)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone, JsonSchema)]
|
||||
pub struct RolloutLine {
|
||||
pub timestamp: String,
|
||||
|
||||
Reference in New Issue
Block a user