Agent jobs (spawn_agents_on_csv) + progress UI (#10935)

## Summary
- Add agent job support: spawn a batch of sub-agents from CSV, auto-run,
auto-export, and store results in SQLite.
- Simplify workflow: remove run/resume/get-status/export tools; spawn is
deterministic and completes in one call.
- Improve exec UX: stable, single-line progress bar with ETA; suppress
sub-agent chatter in exec.

## Why
Enables map-reduce style workflows over arbitrarily large repos using
the existing Codex orchestrator. This addresses review feedback about
overly complex job controls and non-deterministic monitoring.

## Demo (progress bar)
```
./codex-rs/target/debug/codex exec \
  --enable collab \
  --enable sqlite \
  --full-auto \
  --progress-cursor \
  -c agents.max_threads=16 \
  -C /Users/daveaitel/code/codex \
  - <<'PROMPT'
Create /tmp/agent_job_progress_demo.csv with columns: path,area and 30 rows:
path = item-01..item-30, area = test.

Then call spawn_agents_on_csv with:
- csv_path: /tmp/agent_job_progress_demo.csv
- instruction: "Run `python - <<'PY'` to sleep a random 0.3–1.2s, then output JSON with keys: path, score (int). Set score = 1."
- output_csv_path: /tmp/agent_job_progress_demo_out.csv
PROMPT
```

## Review feedback addressed
- Auto-start jobs on spawn; removed run/resume/status/export tools.
- Auto-export on success.
- More descriptive tool spec + clearer prompts.
- Avoid deadlocks on spawn failure; pending/running handled safely.
- Progress bar no longer scrolls; stable single-line redraw.

## Tests
- `cd codex-rs && cargo test -p codex-exec`
- `cd codex-rs && cargo build -p codex-cli`

codex-rs/exec/src/cli.rs

@@ -86,6 +86,10 @@ pub struct Cli {
     #[arg(long = "color", value_enum, default_value_t = Color::Auto)]
     pub color: Color,
 
+    /// Force cursor-based progress updates in exec mode.
+    #[arg(long = "progress-cursor", default_value_t = false)]
+    pub progress_cursor: bool,
+
     /// Print events to stdout as JSONL.
     #[arg(
         long = "json",

codex-rs/exec/src/event_processor_with_human_output.rs

@@ -38,9 +38,12 @@ use codex_utils_elapsed::format_duration;
 use codex_utils_elapsed::format_elapsed;
 use owo_colors::OwoColorize;
 use owo_colors::Style;
+use serde::Deserialize;
 use shlex::try_join;
 use std::collections::HashMap;
+use std::io::Write;
 use std::path::PathBuf;
+use std::time::Duration;
 use std::time::Instant;
 
 use crate::event_processor::CodexStatus;
@@ -76,11 +79,17 @@ pub(crate) struct EventProcessorWithHumanOutput {
     last_total_token_usage: Option<codex_protocol::protocol::TokenUsageInfo>,
     final_message: Option<String>,
     last_proposed_plan: Option<String>,
+    progress_active: bool,
+    progress_last_len: usize,
+    use_ansi_cursor: bool,
+    progress_anchor: bool,
+    progress_done: bool,
 }
 
 impl EventProcessorWithHumanOutput {
     pub(crate) fn create_with_ansi(
         with_ansi: bool,
+        cursor_ansi: bool,
         config: &Config,
         last_message_path: Option<PathBuf>,
     ) -> Self {
@@ -103,6 +112,11 @@ impl EventProcessorWithHumanOutput {
                 last_total_token_usage: None,
                 final_message: None,
                 last_proposed_plan: None,
+                progress_active: false,
+                progress_last_len: 0,
+                use_ansi_cursor: cursor_ansi,
+                progress_anchor: false,
+                progress_done: false,
             }
         } else {
             Self {
@@ -121,11 +135,27 @@ impl EventProcessorWithHumanOutput {
                 last_total_token_usage: None,
                 final_message: None,
                 last_proposed_plan: None,
+                progress_active: false,
+                progress_last_len: 0,
+                use_ansi_cursor: cursor_ansi,
+                progress_anchor: false,
+                progress_done: false,
             }
         }
     }
 }
 
+#[derive(Debug, Deserialize)]
+struct AgentJobProgressMessage {
+    job_id: String,
+    total_items: usize,
+    pending_items: usize,
+    running_items: usize,
+    completed_items: usize,
+    failed_items: usize,
+    eta_seconds: Option<u64>,
+}
+
 struct PatchApplyBegin {
     start_time: Instant,
     auto_approved: bool,
@@ -176,6 +206,18 @@ impl EventProcessor for EventProcessorWithHumanOutput {
 
     fn process_event(&mut self, event: Event) -> CodexStatus {
         let Event { id: _, msg } = event;
+        if let EventMsg::BackgroundEvent(BackgroundEventEvent { message }) = &msg
+            && let Some(update) = Self::parse_agent_job_progress(message)
+        {
+            self.render_agent_job_progress(update);
+            return CodexStatus::Running;
+        }
+        if self.progress_active && !Self::should_interrupt_progress(&msg) {
+            return CodexStatus::Running;
+        }
+        if !Self::is_silent_event(&msg) {
+            self.finish_progress_line();
+        }
         match msg {
             EventMsg::Error(ErrorEvent { message, .. }) => {
                 let prefix = "ERROR:".style(self.red);
@@ -818,6 +860,7 @@ impl EventProcessor for EventProcessorWithHumanOutput {
     }
 
     fn print_final_output(&mut self) {
+        self.finish_progress_line();
         if let Some(usage_info) = &self.last_total_token_usage {
             eprintln!(
                 "{}\n{}",
@@ -841,6 +884,207 @@ impl EventProcessor for EventProcessorWithHumanOutput {
     }
 }
 
+impl EventProcessorWithHumanOutput {
+    fn parse_agent_job_progress(message: &str) -> Option<AgentJobProgressMessage> {
+        let payload = message.strip_prefix("agent_job_progress:")?;
+        serde_json::from_str::<AgentJobProgressMessage>(payload).ok()
+    }
+
+    fn is_silent_event(msg: &EventMsg) -> bool {
+        matches!(
+            msg,
+            EventMsg::ThreadNameUpdated(_)
+                | EventMsg::TokenCount(_)
+                | EventMsg::TurnStarted(_)
+                | EventMsg::ExecApprovalRequest(_)
+                | EventMsg::ApplyPatchApprovalRequest(_)
+                | EventMsg::TerminalInteraction(_)
+                | EventMsg::ExecCommandOutputDelta(_)
+                | EventMsg::GetHistoryEntryResponse(_)
+                | EventMsg::McpListToolsResponse(_)
+                | EventMsg::ListCustomPromptsResponse(_)
+                | EventMsg::ListSkillsResponse(_)
+                | EventMsg::ListRemoteSkillsResponse(_)
+                | EventMsg::RemoteSkillDownloaded(_)
+                | EventMsg::RawResponseItem(_)
+                | EventMsg::UserMessage(_)
+                | EventMsg::EnteredReviewMode(_)
+                | EventMsg::ExitedReviewMode(_)
+                | EventMsg::AgentMessageDelta(_)
+                | EventMsg::AgentReasoningDelta(_)
+                | EventMsg::AgentReasoningRawContentDelta(_)
+                | EventMsg::ItemStarted(_)
+                | EventMsg::ItemCompleted(_)
+                | EventMsg::AgentMessageContentDelta(_)
+                | EventMsg::PlanDelta(_)
+                | EventMsg::ReasoningContentDelta(_)
+                | EventMsg::ReasoningRawContentDelta(_)
+                | EventMsg::SkillsUpdateAvailable
+                | EventMsg::UndoCompleted(_)
+                | EventMsg::UndoStarted(_)
+                | EventMsg::ThreadRolledBack(_)
+                | EventMsg::RequestUserInput(_)
+                | EventMsg::DynamicToolCallRequest(_)
+        )
+    }
+
+    fn should_interrupt_progress(msg: &EventMsg) -> bool {
+        matches!(
+            msg,
+            EventMsg::Error(_)
+                | EventMsg::Warning(_)
+                | EventMsg::DeprecationNotice(_)
+                | EventMsg::StreamError(_)
+                | EventMsg::TurnComplete(_)
+                | EventMsg::ShutdownComplete
+        )
+    }
+
+    fn finish_progress_line(&mut self) {
+        if self.progress_active {
+            self.progress_active = false;
+            self.progress_last_len = 0;
+            self.progress_done = false;
+            if self.use_ansi_cursor {
+                if self.progress_anchor {
+                    eprintln!("\u{1b}[1A\u{1b}[1G\u{1b}[2K");
+                } else {
+                    eprintln!("\u{1b}[1G\u{1b}[2K");
+                }
+            } else {
+                eprintln!();
+            }
+            self.progress_anchor = false;
+        }
+    }
+
+    fn render_agent_job_progress(&mut self, update: AgentJobProgressMessage) {
+        let total = update.total_items.max(1);
+        let processed = update.completed_items + update.failed_items;
+        let percent = (processed as f64 / total as f64 * 100.0).round() as i64;
+        let job_label = update.job_id.chars().take(8).collect::<String>();
+        let eta = update
+            .eta_seconds
+            .map(|secs| format_duration(Duration::from_secs(secs)))
+            .unwrap_or_else(|| "--".to_string());
+        let columns = std::env::var("COLUMNS")
+            .ok()
+            .and_then(|value| value.parse::<usize>().ok())
+            .filter(|value| *value > 0);
+        let line = format_agent_job_progress_line(
+            columns,
+            job_label.as_str(),
+            AgentJobProgressStats {
+                processed,
+                total,
+                percent,
+                failed: update.failed_items,
+                running: update.running_items,
+                pending: update.pending_items,
+            },
+            eta.as_str(),
+        );
+        let done = processed >= update.total_items;
+        if !self.use_ansi_cursor {
+            eprintln!("{line}");
+            if done {
+                self.progress_active = false;
+                self.progress_last_len = 0;
+            }
+            return;
+        }
+        if done && self.progress_done {
+            return;
+        }
+        if !self.progress_active {
+            eprintln!();
+            self.progress_anchor = true;
+            self.progress_done = false;
+        }
+        let mut output = String::new();
+        if self.progress_anchor {
+            output.push_str("\u{1b}[1A\u{1b}[1G\u{1b}[2K");
+        } else {
+            output.push_str("\u{1b}[1G\u{1b}[2K");
+        }
+        output.push_str(&line);
+        if done {
+            output.push('\n');
+            eprint!("{output}");
+            self.progress_active = false;
+            self.progress_last_len = 0;
+            self.progress_anchor = false;
+            self.progress_done = true;
+            return;
+        }
+        eprint!("{output}");
+        let _ = std::io::stderr().flush();
+        self.progress_active = true;
+        self.progress_last_len = line.len();
+    }
+}
+
+struct AgentJobProgressStats {
+    processed: usize,
+    total: usize,
+    percent: i64,
+    failed: usize,
+    running: usize,
+    pending: usize,
+}
+
+fn format_agent_job_progress_line(
+    columns: Option<usize>,
+    job_label: &str,
+    stats: AgentJobProgressStats,
+    eta: &str,
+) -> String {
+    let rest = format!(
+        "{processed}/{total} {percent}% f{failed} r{running} p{pending} eta {eta}",
+        processed = stats.processed,
+        total = stats.total,
+        percent = stats.percent,
+        failed = stats.failed,
+        running = stats.running,
+        pending = stats.pending
+    );
+    let prefix = format!("job {job_label}");
+    let base_len = prefix.len() + rest.len() + 4;
+    let mut bar_width = columns
+        .and_then(|columns| columns.checked_sub(base_len))
+        .filter(|available| *available > 0)
+        .unwrap_or(20usize);
+    let with_bar = |width: usize| {
+        let filled = ((stats.processed as f64 / stats.total as f64) * width as f64)
+            .round()
+            .clamp(0.0, width as f64) as usize;
+        let mut bar = "#".repeat(filled);
+        bar.push_str(&"-".repeat(width - filled));
+        format!("{prefix} [{bar}] {rest}")
+    };
+    let mut line = with_bar(bar_width);
+    if let Some(columns) = columns
+        && line.len() > columns
+    {
+        let min_line = format!("{prefix} {rest}");
+        if min_line.len() > columns {
+            let mut truncated = min_line;
+            if columns > 2 && truncated.len() > columns {
+                truncated.truncate(columns - 2);
+                truncated.push_str("..");
+            }
+            return truncated;
+        }
+        let available = columns.saturating_sub(base_len);
+        if available == 0 {
+            return min_line;
+        }
+        bar_width = available.min(bar_width).max(1);
+        line = with_bar(bar_width);
+    }
+    line
+}
+
 fn escape_command(command: &[String]) -> String {
     try_join(command.iter().map(String::as_str)).unwrap_or_else(|_| command.join(" "))
 }

codex-rs/exec/src/lib.rs

@@ -41,6 +41,7 @@ use codex_protocol::protocol::Op;
 use codex_protocol::protocol::ReviewRequest;
 use codex_protocol::protocol::ReviewTarget;
 use codex_protocol::protocol::SessionSource;
+use codex_protocol::protocol::SubAgentSource;
 use codex_protocol::user_input::UserInput;
 use codex_utils_absolute_path::AbsolutePathBuf;
 use codex_utils_oss::ensure_oss_provider_ready;
@@ -86,6 +87,7 @@ struct ThreadEventEnvelope {
     thread_id: codex_protocol::ThreadId,
     thread: Arc<codex_core::CodexThread>,
     event: Event,
+    suppress_output: bool,
 }
 
 pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option<PathBuf>) -> anyhow::Result<()> {
@@ -113,9 +115,10 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option<PathBuf>) -> any
         prompt,
         output_schema: output_schema_path,
         config_overrides,
+        progress_cursor,
     } = cli;
 
-    let (stdout_with_ansi, stderr_with_ansi) = match color {
+    let (_stdout_with_ansi, stderr_with_ansi) = match color {
         cli::Color::Always => (true, true),
         cli::Color::Never => (false, false),
         cli::Color::Auto => (
@@ -123,6 +126,24 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option<PathBuf>) -> any
             supports_color::on_cached(Stream::Stderr).is_some(),
         ),
     };
+    let cursor_ansi = if progress_cursor {
+        true
+    } else {
+        match color {
+            cli::Color::Never => false,
+            cli::Color::Always => true,
+            cli::Color::Auto => {
+                if stderr_with_ansi || std::io::stderr().is_terminal() {
+                    true
+                } else {
+                    match std::env::var("TERM") {
+                        Ok(term) => !term.is_empty() && term != "dumb",
+                        Err(_) => false,
+                    }
+                }
+            }
+        }
+    };
 
     // Build fmt layer (existing logging) to compose with OTEL layer.
     let default_level = "error";
@@ -318,7 +339,8 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option<PathBuf>) -> any
     let mut event_processor: Box<dyn EventProcessor> = match json_mode {
         true => Box::new(EventProcessorWithJsonOutput::new(last_message_file.clone())),
         _ => Box::new(EventProcessorWithHumanOutput::create_with_ansi(
-            stdout_with_ansi,
+            stderr_with_ansi,
+            cursor_ansi,
             &config,
             last_message_file.clone(),
         )),
@@ -466,7 +488,7 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option<PathBuf>) -> any
 
     let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<ThreadEventEnvelope>();
     let attached_threads = Arc::new(Mutex::new(HashSet::from([primary_thread_id])));
-    spawn_thread_listener(primary_thread_id, thread.clone(), tx.clone());
+    spawn_thread_listener(primary_thread_id, thread.clone(), tx.clone(), false);
 
     {
         let thread = thread.clone();
@@ -494,7 +516,14 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option<PathBuf>) -> any
                         match thread_manager.get_thread(thread_id).await {
                             Ok(thread) => {
                                 attached_threads.lock().await.insert(thread_id);
-                                spawn_thread_listener(thread_id, thread, tx.clone());
+                                let suppress_output =
+                                    is_agent_job_subagent(&thread.config_snapshot().await);
+                                spawn_thread_listener(
+                                    thread_id,
+                                    thread,
+                                    tx.clone(),
+                                    suppress_output,
+                                );
                             }
                             Err(err) => {
                                 warn!("failed to attach listener for thread {thread_id}: {err}")
@@ -549,7 +578,11 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option<PathBuf>) -> any
             thread_id,
             thread,
             event,
+            suppress_output,
         } = envelope;
+        if suppress_output && should_suppress_agent_job_event(&event.msg) {
+            continue;
+        }
         if matches!(event.msg, EventMsg::Error(_)) {
             error_seen = true;
         }
@@ -613,6 +646,7 @@ fn spawn_thread_listener(
     thread_id: codex_protocol::ThreadId,
     thread: Arc<codex_core::CodexThread>,
     tx: tokio::sync::mpsc::UnboundedSender<ThreadEventEnvelope>,
+    suppress_output: bool,
 ) {
     tokio::spawn(async move {
         loop {
@@ -625,6 +659,7 @@ fn spawn_thread_listener(
                         thread_id,
                         thread: Arc::clone(&thread),
                         event,
+                        suppress_output,
                     }) {
                         error!("Error sending event: {err:?}");
                         break;
@@ -645,6 +680,29 @@ fn spawn_thread_listener(
     });
 }
 
+fn is_agent_job_subagent(config: &codex_core::ThreadConfigSnapshot) -> bool {
+    match &config.session_source {
+        SessionSource::SubAgent(SubAgentSource::Other(source)) => source.starts_with("agent_job:"),
+        _ => false,
+    }
+}
+
+fn should_suppress_agent_job_event(msg: &EventMsg) -> bool {
+    !matches!(
+        msg,
+        EventMsg::ExecApprovalRequest(_)
+            | EventMsg::ApplyPatchApprovalRequest(_)
+            | EventMsg::RequestUserInput(_)
+            | EventMsg::DynamicToolCallRequest(_)
+            | EventMsg::ElicitationRequest(_)
+            | EventMsg::Error(_)
+            | EventMsg::Warning(_)
+            | EventMsg::DeprecationNotice(_)
+            | EventMsg::StreamError(_)
+            | EventMsg::ShutdownComplete
+    )
+}
+
 async fn resolve_resume_path(
     config: &Config,
     args: &crate::cli::ResumeArgs,