mirrors/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-04-28 18:32:04 +03:00
Code Issues Packages Projects Releases Wiki Activity
Files
a409c34c856e39ea6b2ac8eb1af7fdf8758c481e
codex/prs/bolinfest/PR-1542.md
Daniel Edrisian d2af202db7 study last 100 days PRs
2025-09-02 15:17:45 -07:00

467 lines
15 KiB
Markdown
Raw Blame History

# PR #1542: Add CLI streaming integration tests
- URL: https://github.com/openai/codex/pull/1542
- Author: aibrahim-oai
- Created: 2025-07-11 20:26:16 UTC
- Updated: 2025-07-13 01:06:04 UTC
- Changes: +127/-0, Files changed: 2, Commits: 6
## Description
## Summary
- add integration test for chat mode streaming via CLI using wiremock
- add integration test for Responses API streaming via fixture
- call `cargo run` to invoke the CLI during tests
## Testing
- `cargo test -p codex-core --test cli_stream -- --nocapture`
- `cargo clippy --all-targets --all-features -- -D warnings`
------
https://chatgpt.com/codex/tasks/task_i_68715980bbec8321999534fdd6a013c1
## Full Diff
```diff
diff --git a/codex-rs/core/tests/cli_responses_fixture.sse b/codex-rs/core/tests/cli_responses_fixture.sse
new file mode 100644
index 0000000000..d297ebafb2
--- /dev/null
+++ b/codex-rs/core/tests/cli_responses_fixture.sse
@@ -0,0 +1,8 @@
+event: response.created
+data: {"type":"response.created","response":{"id":"resp1"}}
+
+event: response.output_item.done
+data: {"type":"response.output_item.done","item":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"fixture hello"}]}}
+
+event: response.completed
+data: {"type":"response.completed","response":{"id":"resp1","output":[]}}
diff --git a/codex-rs/core/tests/cli_stream.rs b/codex-rs/core/tests/cli_stream.rs
new file mode 100644
index 0000000000..df3fedfd48
--- /dev/null
+++ b/codex-rs/core/tests/cli_stream.rs
@@ -0,0 +1,119 @@
+#![expect(clippy::unwrap_used)]
+
+use assert_cmd::Command as AssertCommand;
+use codex_core::exec::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR;
+use tempfile::TempDir;
+use wiremock::Mock;
+use wiremock::MockServer;
+use wiremock::ResponseTemplate;
+use wiremock::matchers::method;
+use wiremock::matchers::path;
+
+/// Tests streaming chat completions through the CLI using a mock server.
+/// This test:
+/// 1. Sets up a mock server that simulates OpenAI's chat completions API
+/// 2. Configures codex to use this mock server via a custom provider
+/// 3. Sends a simple "hello?" prompt and verifies the streamed response
+/// 4. Ensures the response is received exactly once and contains "hi"
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn chat_mode_stream_cli() {
+ if std::env::var(CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR).is_ok() {
+ println!(
+ "Skipping test because it cannot execute when network is disabled in a Codex sandbox."
+ );
+ return;
+ }
+
+ let server = MockServer::start().await;
+ let sse = concat!(
+ "data: {\"choices\":[{\"delta\":{\"content\":\"hi\"}}]}\n\n",
+ "data: {\"choices\":[{\"delta\":{}}]}\n\n",
+ "data: [DONE]\n\n"
+ );
+ Mock::given(method("POST"))
+ .and(path("/v1/chat/completions"))
+ .respond_with(
+ ResponseTemplate::new(200)
+ .insert_header("content-type", "text/event-stream")
+ .set_body_raw(sse, "text/event-stream"),
+ )
+ .expect(1)
+ .mount(&server)
+ .await;
+
+ let home = TempDir::new().unwrap();
+ let provider_override = format!(
+ "model_providers.mock={{ name = \"mock\", base_url = \"{}/v1\", env_key = \"PATH\", wire_api = \"chat\" }}",
+ server.uri()
+ );
+ let mut cmd = AssertCommand::new("cargo");
+ cmd.arg("run")
+ .arg("-p")
+ .arg("codex-cli")
+ .arg("--quiet")
+ .arg("--")
+ .arg("exec")
+ .arg("--skip-git-repo-check")
+ .arg("-c")
+ .arg(&provider_override)
+ .arg("-c")
+ .arg("model_provider=\"mock\"")
+ .arg("-C")
+ .arg(env!("CARGO_MANIFEST_DIR"))
+ .arg("hello?");
+ cmd.env("CODEX_HOME", home.path())
+ .env("OPENAI_API_KEY", "dummy")
+ .env("OPENAI_BASE_URL", format!("{}/v1", server.uri()));
+
+ let output = cmd.output().unwrap();
+ println!("Status: {}", output.status);
+ println!("Stdout:\n{}", String::from_utf8_lossy(&output.stdout));
+ println!("Stderr:\n{}", String::from_utf8_lossy(&output.stderr));
+ assert!(output.status.success());
+ let stdout = String::from_utf8_lossy(&output.stdout);
+ assert!(stdout.contains("hi"));
+ assert_eq!(stdout.matches("hi").count(), 1);
+
+ server.verify().await;
+}
+
+/// Tests streaming responses through the CLI using a local SSE fixture file.
+/// This test:
+/// 1. Uses a pre-recorded SSE response fixture instead of a live server
+/// 2. Configures codex to read from this fixture via CODEX_RS_SSE_FIXTURE env var
+/// 3. Sends a "hello?" prompt and verifies the response
+/// 4. Ensures the fixture content is correctly streamed through the CLI
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn responses_api_stream_cli() {
+ if std::env::var(CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR).is_ok() {
+ println!(
+ "Skipping test because it cannot execute when network is disabled in a Codex sandbox."
+ );
+ return;
+ }
+
+ let fixture =
+ std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/cli_responses_fixture.sse");
+
+ let home = TempDir::new().unwrap();
+ let mut cmd = AssertCommand::new("cargo");
+ cmd.arg("run")
+ .arg("-p")
+ .arg("codex-cli")
+ .arg("--quiet")
+ .arg("--")
+ .arg("exec")
+ .arg("--skip-git-repo-check")
+ .arg("-C")
+ .arg(env!("CARGO_MANIFEST_DIR"))
+ .arg("hello?");
+ cmd.env("CODEX_HOME", home.path())
+ .env("OPENAI_API_KEY", "dummy")
+ .env("CODEX_RS_SSE_FIXTURE", fixture)
+ .env("OPENAI_BASE_URL", "http://unused.local");
+
+ let output = cmd.output().unwrap();
+ assert!(output.status.success());
+ let stdout = String::from_utf8_lossy(&output.stdout);
+ assert!(stdout.contains("fixture hello"));
+}
```
## Review Comments
### codex-rs/core/tests/cli_stream.rs
- Created: 2025-07-12 19:13:30 UTC | Link: https://github.com/openai/codex/pull/1542#discussion_r2202880322
```diff
@@ -0,0 +1,99 @@
+#![expect(clippy::unwrap_used)]
+
+use codex_core::exec::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR;
+use std::process::Command;
+use tempfile::TempDir;
+use wiremock::Mock;
+use wiremock::MockServer;
+use wiremock::ResponseTemplate;
+use wiremock::matchers::method;
+use wiremock::matchers::path;
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn chat_mode_stream_cli() {
+ if std::env::var(CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR).is_ok() {
+ println!(
+ "Skipping test because it cannot execute when network is disabled in a Codex sandbox."
+ );
+ return;
+ }
+
+ let server = MockServer::start().await;
+ let sse = concat!(
```
> I would use `r#` here. Since it isn't a format string, you won't have to escape `{`.
- Created: 2025-07-12 19:14:21 UTC | Link: https://github.com/openai/codex/pull/1542#discussion_r2202880453
```diff
@@ -0,0 +1,99 @@
+#![expect(clippy::unwrap_used)]
+
+use codex_core::exec::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR;
+use std::process::Command;
+use tempfile::TempDir;
+use wiremock::Mock;
+use wiremock::MockServer;
+use wiremock::ResponseTemplate;
+use wiremock::matchers::method;
+use wiremock::matchers::path;
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn chat_mode_stream_cli() {
+ if std::env::var(CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR).is_ok() {
+ println!(
+ "Skipping test because it cannot execute when network is disabled in a Codex sandbox."
+ );
+ return;
+ }
+
+ let server = MockServer::start().await;
+ let sse = concat!(
+ "data: {\"choices\":[{\"delta\":{\"content\":\"hi\"}}]}\n\n",
+ "data: {\"choices\":[{\"delta\":{}}]}\n\n",
+ "data: [DONE]\n\n",
+ );
+ Mock::given(method("POST"))
+ .and(path("/v1/chat/completions"))
+ .respond_with(
+ ResponseTemplate::new(200)
+ .insert_header("content-type", "text/event-stream")
+ .set_body_raw(sse, "text/event-stream"),
+ )
+ .mount(&server)
+ .await;
+
+ let home = TempDir::new().unwrap();
+ let provider_override = format!(
+ "model_providers.mock={{ name = \"mock\", base_url = \"{}/v1\", env_key = \"PATH\", wire_api = \"chat\" }}",
+ server.uri()
+ );
+ let mut cmd = Command::new("cargo");
```
> `Command::cargo_bin("codex-rs")` instead?
- Created: 2025-07-12 19:15:42 UTC | Link: https://github.com/openai/codex/pull/1542#discussion_r2202880670
```diff
@@ -0,0 +1,99 @@
+#![expect(clippy::unwrap_used)]
+
+use codex_core::exec::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR;
+use std::process::Command;
+use tempfile::TempDir;
+use wiremock::Mock;
+use wiremock::MockServer;
+use wiremock::ResponseTemplate;
+use wiremock::matchers::method;
+use wiremock::matchers::path;
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn chat_mode_stream_cli() {
+ if std::env::var(CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR).is_ok() {
+ println!(
+ "Skipping test because it cannot execute when network is disabled in a Codex sandbox."
+ );
+ return;
+ }
+
+ let server = MockServer::start().await;
+ let sse = concat!(
+ "data: {\"choices\":[{\"delta\":{\"content\":\"hi\"}}]}\n\n",
+ "data: {\"choices\":[{\"delta\":{}}]}\n\n",
```
> Is this empty `{}` the way it signals the end of the stream?
- Created: 2025-07-12 19:16:39 UTC | Link: https://github.com/openai/codex/pull/1542#discussion_r2202880874
```diff
@@ -0,0 +1,99 @@
+#![expect(clippy::unwrap_used)]
+
+use codex_core::exec::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR;
+use std::process::Command;
+use tempfile::TempDir;
+use wiremock::Mock;
+use wiremock::MockServer;
+use wiremock::ResponseTemplate;
+use wiremock::matchers::method;
+use wiremock::matchers::path;
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn chat_mode_stream_cli() {
+ if std::env::var(CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR).is_ok() {
+ println!(
+ "Skipping test because it cannot execute when network is disabled in a Codex sandbox."
+ );
+ return;
+ }
+
+ let server = MockServer::start().await;
+ let sse = concat!(
+ "data: {\"choices\":[{\"delta\":{\"content\":\"hi\"}}]}\n\n",
+ "data: {\"choices\":[{\"delta\":{}}]}\n\n",
+ "data: [DONE]\n\n",
+ );
+ Mock::given(method("POST"))
+ .and(path("/v1/chat/completions"))
+ .respond_with(
+ ResponseTemplate::new(200)
+ .insert_header("content-type", "text/event-stream")
+ .set_body_raw(sse, "text/event-stream"),
+ )
+ .mount(&server)
+ .await;
+
+ let home = TempDir::new().unwrap();
+ let provider_override = format!(
+ "model_providers.mock={{ name = \"mock\", base_url = \"{}/v1\", env_key = \"PATH\", wire_api = \"chat\" }}",
+ server.uri()
+ );
+ let mut cmd = Command::new("cargo");
+ cmd.arg("run")
+ .arg("-p")
+ .arg("codex-cli")
+ .arg("--quiet")
+ .arg("--")
+ .arg("exec")
+ .arg("--skip-git-repo-check")
+ .arg("-c")
+ .arg(&provider_override)
+ .arg("-c")
+ .arg("model_provider=\"mock\"")
+ .arg("hello?");
+ cmd.current_dir(env!("CARGO_MANIFEST_DIR"))
+ .env("CODEX_HOME", home.path())
+ .env("OPENAI_API_KEY", "dummy")
+ .env("OPENAI_BASE_URL", format!("{}/v1", server.uri()));
+
+ let output = cmd.output().unwrap();
+ assert!(output.status.success());
+ let stdout = String::from_utf8_lossy(&output.stdout);
+ assert!(stdout.contains("hi"));
+ assert_eq!(stdout.matches("hi").count(), 1);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn responses_api_stream_cli() {
+ if std::env::var(CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR).is_ok() {
+ println!(
+ "Skipping test because it cannot execute when network is disabled in a Codex sandbox."
+ );
+ return;
+ }
+
+ let fixture =
+ std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/cli_responses_fixture.sse");
+
+ let home = TempDir::new().unwrap();
+ let mut cmd = Command::new("cargo");
```
> `Command::cargo_bin` here, as well?
- Created: 2025-07-12 19:17:22 UTC | Link: https://github.com/openai/codex/pull/1542#discussion_r2202881035
```diff
@@ -0,0 +1,99 @@
+#![expect(clippy::unwrap_used)]
+
+use codex_core::exec::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR;
+use std::process::Command;
+use tempfile::TempDir;
+use wiremock::Mock;
+use wiremock::MockServer;
+use wiremock::ResponseTemplate;
+use wiremock::matchers::method;
+use wiremock::matchers::path;
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn chat_mode_stream_cli() {
+ if std::env::var(CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR).is_ok() {
+ println!(
+ "Skipping test because it cannot execute when network is disabled in a Codex sandbox."
+ );
+ return;
+ }
+
+ let server = MockServer::start().await;
+ let sse = concat!(
+ "data: {\"choices\":[{\"delta\":{\"content\":\"hi\"}}]}\n\n",
+ "data: {\"choices\":[{\"delta\":{}}]}\n\n",
+ "data: [DONE]\n\n",
+ );
+ Mock::given(method("POST"))
+ .and(path("/v1/chat/completions"))
+ .respond_with(
+ ResponseTemplate::new(200)
+ .insert_header("content-type", "text/event-stream")
+ .set_body_raw(sse, "text/event-stream"),
+ )
+ .mount(&server)
+ .await;
+
+ let home = TempDir::new().unwrap();
+ let provider_override = format!(
+ "model_providers.mock={{ name = \"mock\", base_url = \"{}/v1\", env_key = \"PATH\", wire_api = \"chat\" }}",
+ server.uri()
+ );
+ let mut cmd = Command::new("cargo");
+ cmd.arg("run")
+ .arg("-p")
+ .arg("codex-cli")
+ .arg("--quiet")
+ .arg("--")
+ .arg("exec")
+ .arg("--skip-git-repo-check")
+ .arg("-c")
+ .arg(&provider_override)
+ .arg("-c")
+ .arg("model_provider=\"mock\"")
+ .arg("hello?");
+ cmd.current_dir(env!("CARGO_MANIFEST_DIR"))
+ .env("CODEX_HOME", home.path())
+ .env("OPENAI_API_KEY", "dummy")
+ .env("OPENAI_BASE_URL", format!("{}/v1", server.uri()));
+
+ let output = cmd.output().unwrap();
+ assert!(output.status.success());
+ let stdout = String::from_utf8_lossy(&output.stdout);
+ assert!(stdout.contains("hi"));
+ assert_eq!(stdout.matches("hi").count(), 1);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn responses_api_stream_cli() {
+ if std::env::var(CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR).is_ok() {
+ println!(
+ "Skipping test because it cannot execute when network is disabled in a Codex sandbox."
+ );
+ return;
+ }
+
+ let fixture =
+ std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/cli_responses_fixture.sse");
+
+ let home = TempDir::new().unwrap();
+ let mut cmd = Command::new("cargo");
+ cmd.arg("run")
+ .arg("-p")
+ .arg("codex-cli")
+ .arg("--quiet")
+ .arg("--")
+ .arg("exec")
+ .arg("--skip-git-repo-check")
+ .arg("hello?");
+ cmd.current_dir(env!("CARGO_MANIFEST_DIR"))
```
> note that if you need `codex` to use a specific `cwd`, it has a `--cd`/`-C` option.
Reference in New Issue View Git Blame Copy Permalink