diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 7306a246cc..bb58e94f6c 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1553,6 +1553,7 @@ dependencies = [ "thiserror 2.0.17", "tiktoken-rs", "tokio", + "tracing", ] [[package]] diff --git a/codex-rs/utils/tokenizer/Cargo.toml b/codex-rs/utils/tokenizer/Cargo.toml index d18d7decc5..9f7e74355c 100644 --- a/codex-rs/utils/tokenizer/Cargo.toml +++ b/codex-rs/utils/tokenizer/Cargo.toml @@ -11,6 +11,7 @@ anyhow = { workspace = true } thiserror = { workspace = true } tiktoken-rs = "0.7" tokio.workspace = true +tracing = { workspace = true } [dev-dependencies] pretty_assertions = { workspace = true } diff --git a/codex-rs/utils/tokenizer/src/lib.rs b/codex-rs/utils/tokenizer/src/lib.rs index fa972d3b44..5946b984f6 100644 --- a/codex-rs/utils/tokenizer/src/lib.rs +++ b/codex-rs/utils/tokenizer/src/lib.rs @@ -7,6 +7,9 @@ use anyhow::Context; use anyhow::Error as AnyhowError; use thiserror::Error; use tiktoken_rs::CoreBPE; +use tracing::error; + +static DEFAULT_TOKENIZER: OnceLock, TokenizerError>> = OnceLock::new(); /// Supported local encodings. #[derive(Debug, Copy, Clone, Eq, PartialEq)] @@ -110,8 +113,6 @@ impl Tokenizer { } } -static DEFAULT_TOKENIZER: OnceLock, TokenizerError>> = OnceLock::new(); - pub fn warm_up_default_tokenizer() { tokio::spawn(tokio::time::timeout(Duration::from_secs(5), async { let _ = shared_default_tokenizer(); @@ -123,12 +124,20 @@ pub fn warm_up_default_tokenizer() { #[must_use] pub fn shared_default_tokenizer() -> Option> { DEFAULT_TOKENIZER - .get_or_init(|| Tokenizer::try_default().map(Arc::new)) + .get_or_init(init_default_tokenizer) .as_ref() .ok() .cloned() } +fn init_default_tokenizer() -> Result, TokenizerError> { + let result = Tokenizer::try_default().map(Arc::new); + if let Err(ref error) = result { + error!("failed to initialize default tokenizer: {error}"); + } + result +} + #[cfg(test)] mod tests { use super::*;