move

codex-rs/utils/tokenizer/Cargo.toml

@@ -10,6 +10,7 @@ workspace = true
 anyhow = { workspace = true }
 thiserror = { workspace = true }
 tiktoken-rs = "0.7"
+tokio.workspace = true
 
 [dev-dependencies]
 pretty_assertions = { workspace = true }

codex-rs/utils/tokenizer/src/lib.rs

@@ -1,6 +1,7 @@
 use std::fmt;
 use std::sync::Arc;
 use std::sync::OnceLock;
+use std::time::Duration;
 
 use anyhow::Context;
 use anyhow::Error as AnyhowError;
@@ -111,6 +112,12 @@ impl Tokenizer {
 
 static DEFAULT_TOKENIZER: OnceLock<Result<Arc<Tokenizer>, TokenizerError>> = OnceLock::new();
 
+pub fn warm_up_default_tokenizer() {
+    tokio::spawn(tokio::time::timeout(Duration::from_secs(5), async {
+        let _ = shared_default_tokenizer();
+    }));
+}
+
 /// Return a shared default tokenizer (`O200kBase`), loading it once per process.
 /// Returns `None` if initialization fails.
 #[must_use]