diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 7eb8af5c7f..81cb7f5f4c 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -2103,6 +2103,7 @@ dependencies = [ "codex-cloud-tasks", "codex-config", "codex-core", + "codex-core-plugins", "codex-exec", "codex-exec-server", "codex-execpolicy", @@ -2444,7 +2445,6 @@ dependencies = [ "whoami", "windows-sys 0.52.0", "wiremock", - "zip 2.4.2", "zstd 0.13.3", ] @@ -2452,6 +2452,7 @@ dependencies = [ name = "codex-core-plugins" version = "0.0.0" dependencies = [ + "anyhow", "chrono", "codex-app-server-protocol", "codex-config", @@ -2459,11 +2460,13 @@ dependencies = [ "codex-exec-server", "codex-git-utils", "codex-login", + "codex-otel", "codex-plugin", "codex-protocol", "codex-utils-absolute-path", "codex-utils-plugins", "dirs", + "libc", "pretty_assertions", "reqwest", "serde", @@ -2474,6 +2477,8 @@ dependencies = [ "toml 0.9.11+spec-1.1.0", "tracing", "url", + "wiremock", + "zip 2.4.2", ] [[package]] @@ -3421,6 +3426,7 @@ dependencies = [ "codex-cloud-requirements", "codex-config", "codex-connectors", + "codex-core-plugins", "codex-core-skills", "codex-exec-server", "codex-features", diff --git a/codex-rs/app-server-client/src/lib.rs b/codex-rs/app-server-client/src/lib.rs index 1c3de1208b..e94fc87012 100644 --- a/codex-rs/app-server-client/src/lib.rs +++ b/codex-rs/app-server-client/src/lib.rs @@ -98,7 +98,7 @@ pub mod legacy_core { } pub mod plugins { - pub use codex_core::plugins::*; + pub use codex_core::plugins::PluginsManager; } pub mod review_format { diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 197552782f..a5f2eb4001 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -245,27 +245,28 @@ use codex_core::find_thread_name_by_id; use codex_core::find_thread_names_by_ids; use codex_core::find_thread_path_by_id_str; use codex_core::path_utils; -use codex_core::plugins::MarketplaceAddError; -use codex_core::plugins::MarketplaceRemoveError; -use codex_core::plugins::MarketplaceRemoveRequest as CoreMarketplaceRemoveRequest; -use codex_core::plugins::OPENAI_CURATED_MARKETPLACE_NAME; use codex_core::plugins::PluginInstallError as CorePluginInstallError; use codex_core::plugins::PluginInstallRequest; use codex_core::plugins::PluginReadRequest; use codex_core::plugins::PluginUninstallError as CorePluginUninstallError; -use codex_core::plugins::add_marketplace as add_marketplace_to_codex_home; -use codex_core::plugins::remove_marketplace; use codex_core::read_head_for_summary; use codex_core::read_session_meta_line; use codex_core::sandboxing::SandboxPermissions; use codex_core::windows_sandbox::WindowsSandboxLevelExt; use codex_core::windows_sandbox::WindowsSandboxSetupMode as CoreWindowsSandboxSetupMode; use codex_core::windows_sandbox::WindowsSandboxSetupRequest; +use codex_core_plugins::OPENAI_CURATED_MARKETPLACE_NAME; use codex_core_plugins::loader::load_plugin_apps; use codex_core_plugins::loader::load_plugin_mcp_servers; use codex_core_plugins::manifest::PluginManifestInterface; use codex_core_plugins::marketplace::MarketplaceError; use codex_core_plugins::marketplace::MarketplacePluginSource; +use codex_core_plugins::marketplace_add::MarketplaceAddError; +use codex_core_plugins::marketplace_add::MarketplaceAddRequest; +use codex_core_plugins::marketplace_add::add_marketplace as add_marketplace_to_codex_home; +use codex_core_plugins::marketplace_remove::MarketplaceRemoveError; +use codex_core_plugins::marketplace_remove::MarketplaceRemoveRequest as CoreMarketplaceRemoveRequest; +use codex_core_plugins::marketplace_remove::remove_marketplace; use codex_core_plugins::remote::RemoteMarketplace; use codex_core_plugins::remote::RemotePluginCatalogError; use codex_core_plugins::remote::RemotePluginDetail as RemoteCatalogPluginDetail; @@ -6778,7 +6779,7 @@ impl CodexMessageProcessor { async fn marketplace_add(&self, request_id: ConnectionRequestId, params: MarketplaceAddParams) { let result = add_marketplace_to_codex_home( self.config.codex_home.to_path_buf(), - codex_core::plugins::MarketplaceAddRequest { + MarketplaceAddRequest { source: params.source, ref_name: params.ref_name, sparse_paths: params.sparse_paths.unwrap_or_default(), diff --git a/codex-rs/app-server/src/config/external_agent_config.rs b/codex-rs/app-server/src/config/external_agent_config.rs index 1e57cd4a4d..9ecf06bcf0 100644 --- a/codex-rs/app-server/src/config/external_agent_config.rs +++ b/codex-rs/app-server/src/config/external_agent_config.rs @@ -1,14 +1,14 @@ use codex_config::types::PluginConfig; use codex_core::config::Config; use codex_core::config::ConfigBuilder; -use codex_core::plugins::MarketplaceAddRequest; use codex_core::plugins::PluginId; use codex_core::plugins::PluginInstallRequest; use codex_core::plugins::PluginsManager; -use codex_core::plugins::add_marketplace; -use codex_core::plugins::is_local_marketplace_source; use codex_core_plugins::marketplace::MarketplacePluginInstallPolicy; use codex_core_plugins::marketplace::find_marketplace_manifest_path; +use codex_core_plugins::marketplace_add::MarketplaceAddRequest; +use codex_core_plugins::marketplace_add::add_marketplace; +use codex_core_plugins::marketplace_add::is_local_marketplace_source; use codex_protocol::protocol::Product; use serde_json::Value as JsonValue; use std::collections::BTreeMap; diff --git a/codex-rs/app-server/tests/suite/v2/marketplace_remove.rs b/codex-rs/app-server/tests/suite/v2/marketplace_remove.rs index fb144001e4..dc438499f3 100644 --- a/codex-rs/app-server/tests/suite/v2/marketplace_remove.rs +++ b/codex-rs/app-server/tests/suite/v2/marketplace_remove.rs @@ -10,7 +10,7 @@ use codex_app_server_protocol::MarketplaceRemoveResponse; use codex_app_server_protocol::RequestId; use codex_config::MarketplaceConfigUpdate; use codex_config::record_user_marketplace; -use codex_core::plugins::marketplace_install_root; +use codex_core_plugins::installed_marketplaces::marketplace_install_root; use pretty_assertions::assert_eq; use tempfile::TempDir; use tokio::time::timeout; diff --git a/codex-rs/cli/Cargo.toml b/codex-rs/cli/Cargo.toml index 4bd0923533..d318297f8f 100644 --- a/codex-rs/cli/Cargo.toml +++ b/codex-rs/cli/Cargo.toml @@ -30,6 +30,7 @@ codex-cloud-tasks = { path = "../cloud-tasks" } codex-utils-cli = { workspace = true } codex-config = { workspace = true } codex-core = { workspace = true } +codex-core-plugins = { workspace = true } codex-exec = { workspace = true } codex-exec-server = { workspace = true } codex-execpolicy = { workspace = true } diff --git a/codex-rs/cli/src/marketplace_cmd.rs b/codex-rs/cli/src/marketplace_cmd.rs index 5130e9ebcd..d8756c263b 100644 --- a/codex-rs/cli/src/marketplace_cmd.rs +++ b/codex-rs/cli/src/marketplace_cmd.rs @@ -4,12 +4,12 @@ use anyhow::bail; use clap::Parser; use codex_core::config::Config; use codex_core::config::find_codex_home; -use codex_core::plugins::MarketplaceAddRequest; -use codex_core::plugins::MarketplaceRemoveRequest; use codex_core::plugins::PluginMarketplaceUpgradeOutcome; use codex_core::plugins::PluginsManager; -use codex_core::plugins::add_marketplace; -use codex_core::plugins::remove_marketplace; +use codex_core_plugins::marketplace_add::MarketplaceAddRequest; +use codex_core_plugins::marketplace_add::add_marketplace; +use codex_core_plugins::marketplace_remove::MarketplaceRemoveRequest; +use codex_core_plugins::marketplace_remove::remove_marketplace; use codex_utils_cli::CliConfigOverrides; #[derive(Debug, Parser)] diff --git a/codex-rs/cli/tests/marketplace_add.rs b/codex-rs/cli/tests/marketplace_add.rs index e4256f52b2..5ab18e24c4 100644 --- a/codex-rs/cli/tests/marketplace_add.rs +++ b/codex-rs/cli/tests/marketplace_add.rs @@ -1,6 +1,6 @@ use anyhow::Result; use codex_config::CONFIG_TOML_FILE; -use codex_core::plugins::marketplace_install_root; +use codex_core_plugins::installed_marketplaces::marketplace_install_root; use predicates::str::contains; use pretty_assertions::assert_eq; use std::path::Path; diff --git a/codex-rs/cli/tests/marketplace_remove.rs b/codex-rs/cli/tests/marketplace_remove.rs index 06e213bae6..5c8c7a1f91 100644 --- a/codex-rs/cli/tests/marketplace_remove.rs +++ b/codex-rs/cli/tests/marketplace_remove.rs @@ -1,7 +1,7 @@ use anyhow::Result; use codex_config::MarketplaceConfigUpdate; use codex_config::record_user_marketplace; -use codex_core::plugins::marketplace_install_root; +use codex_core_plugins::installed_marketplaces::marketplace_install_root; use predicates::str::contains; use std::path::Path; use tempfile::TempDir; diff --git a/codex-rs/core-plugins/Cargo.toml b/codex-rs/core-plugins/Cargo.toml index 0372d9a14a..036b160365 100644 --- a/codex-rs/core-plugins/Cargo.toml +++ b/codex-rs/core-plugins/Cargo.toml @@ -19,6 +19,7 @@ codex-core-skills = { workspace = true } codex-exec-server = { workspace = true } codex-git-utils = { workspace = true } codex-login = { workspace = true } +codex-otel = { workspace = true } codex-plugin = { workspace = true } codex-protocol = { workspace = true } codex-utils-absolute-path = { workspace = true } @@ -30,11 +31,15 @@ serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } tempfile = { workspace = true } thiserror = { workspace = true } -tokio = { workspace = true, features = ["fs"] } +tokio = { workspace = true, features = ["fs", "macros", "rt", "time"] } toml = { workspace = true } tracing = { workspace = true } url = { workspace = true } +zip = { workspace = true } [dev-dependencies] +anyhow = { workspace = true } +libc = { workspace = true } pretty_assertions = { workspace = true } tempfile = { workspace = true } +wiremock = { workspace = true } diff --git a/codex-rs/core/src/plugins/installed_marketplaces.rs b/codex-rs/core-plugins/src/installed_marketplaces.rs similarity index 86% rename from codex-rs/core/src/plugins/installed_marketplaces.rs rename to codex-rs/core-plugins/src/installed_marketplaces.rs index a6f80139a9..f313a06276 100644 --- a/codex-rs/core/src/plugins/installed_marketplaces.rs +++ b/codex-rs/core-plugins/src/installed_marketplaces.rs @@ -1,11 +1,12 @@ -use crate::config::Config; -use codex_core_plugins::marketplace::find_marketplace_manifest_path; -use codex_utils_absolute_path::AbsolutePathBuf; use std::path::Path; use std::path::PathBuf; + +use codex_config::ConfigLayerStack; +use codex_plugin::validate_plugin_segment; +use codex_utils_absolute_path::AbsolutePathBuf; use tracing::warn; -use super::validate_plugin_segment; +use crate::marketplace::find_marketplace_manifest_path; pub const INSTALLED_MARKETPLACES_DIR: &str = ".tmp/marketplaces"; @@ -13,11 +14,11 @@ pub fn marketplace_install_root(codex_home: &Path) -> PathBuf { codex_home.join(INSTALLED_MARKETPLACES_DIR) } -pub(crate) fn installed_marketplace_roots_from_config( - config: &Config, +pub fn installed_marketplace_roots_from_layer_stack( + config_layer_stack: &ConfigLayerStack, codex_home: &Path, ) -> Vec { - let Some(user_layer) = config.config_layer_stack.get_user_layer() else { + let Some(user_layer) = config_layer_stack.get_user_layer() else { return Vec::new(); }; let Some(marketplaces_value) = user_layer.config.get("marketplaces") else { @@ -59,7 +60,7 @@ pub(crate) fn installed_marketplace_roots_from_config( roots } -pub(crate) fn resolve_configured_marketplace_root( +pub fn resolve_configured_marketplace_root( marketplace_name: &str, marketplace: &toml::Value, default_install_root: &Path, diff --git a/codex-rs/core-plugins/src/lib.rs b/codex-rs/core-plugins/src/lib.rs index ffb45bc612..61e5ef3724 100644 --- a/codex-rs/core-plugins/src/lib.rs +++ b/codex-rs/core-plugins/src/lib.rs @@ -1,8 +1,15 @@ +pub mod installed_marketplaces; pub mod loader; pub mod manifest; pub mod marketplace; +pub mod marketplace_add; +pub mod marketplace_remove; pub mod marketplace_upgrade; pub mod remote; pub mod remote_legacy; +pub mod startup_sync; pub mod store; pub mod toggles; + +pub const OPENAI_CURATED_MARKETPLACE_NAME: &str = "openai-curated"; +pub const OPENAI_BUNDLED_MARKETPLACE_NAME: &str = "openai-bundled"; diff --git a/codex-rs/core-plugins/src/loader.rs b/codex-rs/core-plugins/src/loader.rs index 21c5aa6b43..32d0bec7af 100644 --- a/codex-rs/core-plugins/src/loader.rs +++ b/codex-rs/core-plugins/src/loader.rs @@ -1,3 +1,4 @@ +use crate::OPENAI_CURATED_MARKETPLACE_NAME; use crate::manifest::PluginManifestPaths; use crate::manifest::load_plugin_manifest; use crate::marketplace::MarketplacePluginSource; @@ -40,7 +41,6 @@ use tracing::warn; const DEFAULT_SKILLS_DIR_NAME: &str = "skills"; const DEFAULT_MCP_CONFIG_FILE: &str = ".mcp.json"; const DEFAULT_APP_CONFIG_FILE: &str = ".app.json"; -const OPENAI_CURATED_MARKETPLACE_NAME: &str = "openai-curated"; const CONFIG_TOML_FILE: &str = "config.toml"; #[derive(Clone, Copy, PartialEq, Eq)] diff --git a/codex-rs/core/src/plugins/marketplace_add.rs b/codex-rs/core-plugins/src/marketplace_add.rs similarity index 99% rename from codex-rs/core/src/plugins/marketplace_add.rs rename to codex-rs/core-plugins/src/marketplace_add.rs index a03a2134e5..aeea5872d9 100644 --- a/codex-rs/core/src/plugins/marketplace_add.rs +++ b/codex-rs/core-plugins/src/marketplace_add.rs @@ -1,5 +1,5 @@ -use super::OPENAI_CURATED_MARKETPLACE_NAME; -use super::marketplace_install_root; +use crate::OPENAI_CURATED_MARKETPLACE_NAME; +use crate::installed_marketplaces::marketplace_install_root; use codex_utils_absolute_path::AbsolutePathBuf; use std::fs; use std::path::Path; diff --git a/codex-rs/core/src/plugins/marketplace_add/install.rs b/codex-rs/core-plugins/src/marketplace_add/install.rs similarity index 100% rename from codex-rs/core/src/plugins/marketplace_add/install.rs rename to codex-rs/core-plugins/src/marketplace_add/install.rs diff --git a/codex-rs/core/src/plugins/marketplace_add/metadata.rs b/codex-rs/core-plugins/src/marketplace_add/metadata.rs similarity index 98% rename from codex-rs/core/src/plugins/marketplace_add/metadata.rs rename to codex-rs/core-plugins/src/marketplace_add/metadata.rs index 06b5e39564..66b17f6c40 100644 --- a/codex-rs/core/src/plugins/marketplace_add/metadata.rs +++ b/codex-rs/core-plugins/src/marketplace_add/metadata.rs @@ -1,10 +1,10 @@ use super::MarketplaceAddError; use super::source::MarketplaceSource; -use crate::plugins::installed_marketplaces::resolve_configured_marketplace_root; +use crate::installed_marketplaces::resolve_configured_marketplace_root; +use crate::marketplace::validate_marketplace_root; use codex_config::CONFIG_TOML_FILE; use codex_config::MarketplaceConfigUpdate; use codex_config::record_user_marketplace; -use codex_core_plugins::marketplace::validate_marketplace_root; use std::fs; use std::io::ErrorKind; use std::path::Path; diff --git a/codex-rs/core/src/plugins/marketplace_add/source.rs b/codex-rs/core-plugins/src/marketplace_add/source.rs similarity index 99% rename from codex-rs/core/src/plugins/marketplace_add/source.rs rename to codex-rs/core-plugins/src/marketplace_add/source.rs index e723c8865a..fdcbd4094e 100644 --- a/codex-rs/core/src/plugins/marketplace_add/source.rs +++ b/codex-rs/core-plugins/src/marketplace_add/source.rs @@ -1,6 +1,6 @@ use super::MarketplaceAddError; -use crate::plugins::validate_plugin_segment; -use codex_core_plugins::marketplace::validate_marketplace_root; +use crate::marketplace::validate_marketplace_root; +use codex_plugin::validate_plugin_segment; use std::path::Path; use std::path::PathBuf; diff --git a/codex-rs/core/src/plugins/marketplace_remove.rs b/codex-rs/core-plugins/src/marketplace_remove.rs similarity index 99% rename from codex-rs/core/src/plugins/marketplace_remove.rs rename to codex-rs/core-plugins/src/marketplace_remove.rs index 490b16f95d..aa5a5078aa 100644 --- a/codex-rs/core/src/plugins/marketplace_remove.rs +++ b/codex-rs/core-plugins/src/marketplace_remove.rs @@ -1,7 +1,7 @@ -use crate::plugins::marketplace_install_root; -use crate::plugins::validate_plugin_segment; +use crate::installed_marketplaces::marketplace_install_root; use codex_config::RemoveMarketplaceConfigOutcome; use codex_config::remove_user_marketplace_config; +use codex_plugin::validate_plugin_segment; use codex_utils_absolute_path::AbsolutePathBuf; use std::fs; use std::path::Path; diff --git a/codex-rs/core-plugins/src/startup_sync.rs b/codex-rs/core-plugins/src/startup_sync.rs new file mode 100644 index 0000000000..f03aff93f6 --- /dev/null +++ b/codex-rs/core-plugins/src/startup_sync.rs @@ -0,0 +1,938 @@ +use std::path::Path; +use std::path::PathBuf; +use std::process::Command; +use std::process::Output; +use std::process::Stdio; +use std::time::Duration; + +use codex_otel::CURATED_PLUGINS_STARTUP_SYNC_FINAL_METRIC; +use codex_otel::CURATED_PLUGINS_STARTUP_SYNC_METRIC; +use reqwest::Client; +use serde::Deserialize; +use tempfile::TempDir; +use tracing::warn; +use zip::ZipArchive; + +use codex_login::default_client::build_reqwest_client; + +const GITHUB_API_BASE_URL: &str = "https://api.github.com"; +const GITHUB_API_ACCEPT_HEADER: &str = "application/vnd.github+json"; +const GITHUB_API_VERSION_HEADER: &str = "2022-11-28"; +const CURATED_PLUGINS_BACKUP_ARCHIVE_API_URL: &str = + "https://chatgpt.com/backend-api/plugins/export/curated"; +const OPENAI_PLUGINS_OWNER: &str = "openai"; +const OPENAI_PLUGINS_REPO: &str = "plugins"; +const CURATED_PLUGINS_RELATIVE_DIR: &str = ".tmp/plugins"; +const CURATED_PLUGINS_SHA_FILE: &str = ".tmp/plugins.sha"; +const CURATED_PLUGINS_BACKUP_ARCHIVE_FALLBACK_VERSION: &str = "export-backup"; +const CURATED_PLUGINS_GIT_TIMEOUT: Duration = Duration::from_secs(30); +const CURATED_PLUGINS_HTTP_TIMEOUT: Duration = Duration::from_secs(30); +const CURATED_PLUGINS_BACKUP_ARCHIVE_TIMEOUT: Duration = Duration::from_secs(30); +// Keep this comfortably above a normal sync attempt so we do not race another Codex process. +const CURATED_PLUGINS_STALE_TEMP_DIR_MAX_AGE: Duration = Duration::from_secs(10 * 60); + +#[derive(Debug, Deserialize)] +struct GitHubRepositorySummary { + default_branch: String, +} + +#[derive(Debug, Deserialize)] +struct GitHubGitRefSummary { + object: GitHubGitRefObject, +} + +#[derive(Debug, Deserialize)] +struct GitHubGitRefObject { + sha: String, +} + +#[derive(Debug, Deserialize)] +struct CuratedPluginsBackupArchiveResponse { + download_url: String, +} + +pub fn curated_plugins_repo_path(codex_home: &Path) -> PathBuf { + codex_home.join(CURATED_PLUGINS_RELATIVE_DIR) +} + +pub fn read_curated_plugins_sha(codex_home: &Path) -> Option { + read_sha_file(curated_plugins_sha_path(codex_home).as_path()) +} + +fn curated_plugins_sha_path(codex_home: &Path) -> PathBuf { + codex_home.join(CURATED_PLUGINS_SHA_FILE) +} + +pub fn sync_openai_plugins_repo(codex_home: &Path) -> Result { + sync_openai_plugins_repo_with_transport_overrides( + codex_home, + "git", + GITHUB_API_BASE_URL, + CURATED_PLUGINS_BACKUP_ARCHIVE_API_URL, + ) +} + +fn sync_openai_plugins_repo_with_transport_overrides( + codex_home: &Path, + git_binary: &str, + api_base_url: &str, + backup_archive_api_url: &str, +) -> Result { + match sync_openai_plugins_repo_via_git(codex_home, git_binary) { + Ok(remote_sha) => { + emit_curated_plugins_startup_sync_metric("git", "success"); + emit_curated_plugins_startup_sync_final_metric("git", "success"); + Ok(remote_sha) + } + Err(err) => { + emit_curated_plugins_startup_sync_metric("git", "failure"); + warn!( + error = %err, + git_binary, + "git sync failed for curated plugin sync; falling back to GitHub HTTP" + ); + match sync_openai_plugins_repo_via_http(codex_home, api_base_url) { + Ok(remote_sha) => { + emit_curated_plugins_startup_sync_metric("http", "success"); + emit_curated_plugins_startup_sync_final_metric("http", "success"); + Ok(remote_sha) + } + Err(http_err) => { + emit_curated_plugins_startup_sync_metric("http", "failure"); + if has_local_curated_plugins_snapshot(codex_home) { + emit_curated_plugins_startup_sync_final_metric("http", "failure"); + warn!( + error = %http_err, + "GitHub HTTP sync failed for curated plugin sync; skipping export archive fallback because a local curated plugins snapshot already exists" + ); + Err(format!( + "git sync failed for curated plugin sync: {err}; GitHub HTTP sync failed for curated plugin sync: {http_err}; export archive fallback skipped because a local curated plugins snapshot already exists" + )) + } else { + // The export archive is a lagging backup path. Only use it to bootstrap a + // missing local curated snapshot, never to refresh an existing one. + warn!( + error = %http_err, + backup_archive_api_url, + "GitHub HTTP sync failed for curated plugin sync; falling back to export archive" + ); + let result = sync_openai_plugins_repo_via_backup_archive( + codex_home, + backup_archive_api_url, + ); + let status = if result.is_ok() { "success" } else { "failure" }; + emit_curated_plugins_startup_sync_metric("export_archive", status); + emit_curated_plugins_startup_sync_final_metric("export_archive", status); + result.map_err(|export_err| { + format!( + "git sync failed for curated plugin sync: {err}; GitHub HTTP sync failed for curated plugin sync: {http_err}; export archive sync failed for curated plugin sync: {export_err}" + ) + }) + } + } + } + } + } +} + +fn sync_openai_plugins_repo_via_git(codex_home: &Path, git_binary: &str) -> Result { + let repo_path = curated_plugins_repo_path(codex_home); + let sha_path = codex_home.join(CURATED_PLUGINS_SHA_FILE); + let remote_sha = git_ls_remote_head_sha(git_binary)?; + let local_sha = read_local_git_or_sha_file(&repo_path, &sha_path, git_binary); + + if local_sha.as_deref() == Some(remote_sha.as_str()) && repo_path.join(".git").is_dir() { + return Ok(remote_sha); + } + + let staged_repo_dir = prepare_curated_repo_parent_and_temp_dir(&repo_path)?; + let clone_output = run_git_command_with_timeout( + Command::new(git_binary) + .env("GIT_OPTIONAL_LOCKS", "0") + .arg("clone") + .arg("--depth") + .arg("1") + .arg("https://github.com/openai/plugins.git") + .arg(staged_repo_dir.path()), + "git clone curated plugins repo", + CURATED_PLUGINS_GIT_TIMEOUT, + )?; + ensure_git_success(&clone_output, "git clone curated plugins repo")?; + + let cloned_sha = git_head_sha(staged_repo_dir.path(), git_binary)?; + if cloned_sha != remote_sha { + return Err(format!( + "curated plugins clone HEAD mismatch: expected {remote_sha}, got {cloned_sha}" + )); + } + + ensure_marketplace_manifest_exists(staged_repo_dir.path())?; + activate_curated_repo(&repo_path, staged_repo_dir)?; + write_curated_plugins_sha(&sha_path, &remote_sha)?; + Ok(remote_sha) +} + +fn sync_openai_plugins_repo_via_http( + codex_home: &Path, + api_base_url: &str, +) -> Result { + let repo_path = curated_plugins_repo_path(codex_home); + let sha_path = codex_home.join(CURATED_PLUGINS_SHA_FILE); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|err| format!("failed to create curated plugins sync runtime: {err}"))?; + let remote_sha = runtime.block_on(fetch_curated_repo_remote_sha(api_base_url))?; + let local_sha = read_sha_file(&sha_path); + + if local_sha.as_deref() == Some(remote_sha.as_str()) && repo_path.is_dir() { + return Ok(remote_sha); + } + + let staged_repo_dir = prepare_curated_repo_parent_and_temp_dir(&repo_path)?; + let zipball_bytes = runtime.block_on(fetch_curated_repo_zipball(api_base_url, &remote_sha))?; + extract_zipball_to_dir(&zipball_bytes, staged_repo_dir.path())?; + ensure_marketplace_manifest_exists(staged_repo_dir.path())?; + activate_curated_repo(&repo_path, staged_repo_dir)?; + write_curated_plugins_sha(&sha_path, &remote_sha)?; + Ok(remote_sha) +} + +fn sync_openai_plugins_repo_via_backup_archive( + codex_home: &Path, + backup_archive_api_url: &str, +) -> Result { + let repo_path = curated_plugins_repo_path(codex_home); + let sha_path = curated_plugins_sha_path(codex_home); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|err| format!("failed to create curated plugins sync runtime: {err}"))?; + let staged_repo_dir = prepare_curated_repo_parent_and_temp_dir(&repo_path)?; + let zipball_bytes = runtime.block_on(fetch_curated_repo_backup_archive_zip( + backup_archive_api_url, + ))?; + extract_zipball_to_dir(&zipball_bytes, staged_repo_dir.path())?; + ensure_marketplace_manifest_exists(staged_repo_dir.path())?; + let export_version = read_extracted_backup_archive_git_sha(staged_repo_dir.path())? + .unwrap_or_else(|| CURATED_PLUGINS_BACKUP_ARCHIVE_FALLBACK_VERSION.to_string()); + activate_curated_repo(&repo_path, staged_repo_dir)?; + write_curated_plugins_sha(&sha_path, &export_version)?; + Ok(export_version) +} + +pub fn has_local_curated_plugins_snapshot(codex_home: &Path) -> bool { + curated_plugins_repo_path(codex_home) + .join(".agents/plugins/marketplace.json") + .is_file() + && codex_home.join(CURATED_PLUGINS_SHA_FILE).is_file() +} + +fn prepare_curated_repo_parent_and_temp_dir(repo_path: &Path) -> Result { + let Some(parent) = repo_path.parent() else { + return Err(format!( + "failed to determine curated plugins parent directory for {}", + repo_path.display() + )); + }; + std::fs::create_dir_all(parent).map_err(|err| { + format!( + "failed to create curated plugins parent directory {}: {err}", + parent.display() + ) + })?; + remove_stale_curated_repo_temp_dirs(parent, CURATED_PLUGINS_STALE_TEMP_DIR_MAX_AGE); + + let clone_dir = tempfile::Builder::new() + .prefix("plugins-clone-") + .tempdir_in(parent) + .map_err(|err| { + format!( + "failed to create temporary curated plugins directory in {}: {err}", + parent.display() + ) + })?; + Ok(clone_dir) +} + +fn remove_stale_curated_repo_temp_dirs(parent: &Path, max_age: Duration) { + let entries = match std::fs::read_dir(parent) { + Ok(entries) => entries, + Err(err) => { + warn!( + error = %err, + parent = %parent.display(), + "failed to list curated plugins temp directory parent for stale cleanup" + ); + return; + } + }; + + for entry in entries.flatten() { + let file_type = match entry.file_type() { + Ok(file_type) => file_type, + Err(err) => { + warn!( + error = %err, + path = %entry.path().display(), + "failed to inspect curated plugins temp directory entry" + ); + continue; + } + }; + if !file_type.is_dir() { + continue; + } + + let path = entry.path(); + let is_plugins_clone_dir = path + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with("plugins-clone-")); + if !is_plugins_clone_dir { + continue; + } + + let metadata = match entry.metadata() { + Ok(metadata) => metadata, + Err(err) => { + warn!( + error = %err, + path = %path.display(), + "failed to read curated plugins temp directory metadata" + ); + continue; + } + }; + let modified = match metadata.modified() { + Ok(modified) => modified, + Err(err) => { + warn!( + error = %err, + path = %path.display(), + "failed to read curated plugins temp directory modification time" + ); + continue; + } + }; + let age = match modified.elapsed() { + Ok(age) => age, + Err(err) => { + warn!( + error = %err, + path = %path.display(), + "failed to compute curated plugins temp directory age" + ); + continue; + } + }; + if age < max_age { + continue; + } + + if let Err(err) = std::fs::remove_dir_all(&path) { + warn!( + error = %err, + path = %path.display(), + "failed to remove stale curated plugins temp directory" + ); + } + } +} + +fn emit_curated_plugins_startup_sync_metric(transport: &'static str, status: &'static str) { + emit_curated_plugins_startup_sync_counter( + CURATED_PLUGINS_STARTUP_SYNC_METRIC, + transport, + status, + ); +} + +fn emit_curated_plugins_startup_sync_final_metric(transport: &'static str, status: &'static str) { + emit_curated_plugins_startup_sync_counter( + CURATED_PLUGINS_STARTUP_SYNC_FINAL_METRIC, + transport, + status, + ); +} + +fn emit_curated_plugins_startup_sync_counter( + metric_name: &str, + transport: &'static str, + status: &'static str, +) { + let Some(metrics) = codex_otel::global() else { + return; + }; + let tags = [("transport", transport), ("status", status)]; + let _ = metrics.counter(metric_name, /*inc*/ 1, &tags); +} + +fn ensure_marketplace_manifest_exists(repo_path: &Path) -> Result<(), String> { + if repo_path.join(".agents/plugins/marketplace.json").is_file() { + return Ok(()); + } + Err(format!( + "curated plugins archive missing marketplace manifest at {}", + repo_path.join(".agents/plugins/marketplace.json").display() + )) +} + +fn activate_curated_repo(repo_path: &Path, staged_repo_dir: TempDir) -> Result<(), String> { + let staged_repo_path = staged_repo_dir.path(); + if repo_path.exists() { + let parent = repo_path.parent().ok_or_else(|| { + format!( + "failed to determine curated plugins parent directory for {}", + repo_path.display() + ) + })?; + let backup_dir = tempfile::Builder::new() + .prefix("plugins-backup-") + .tempdir_in(parent) + .map_err(|err| { + format!( + "failed to create curated plugins backup directory in {}: {err}", + parent.display() + ) + })?; + let backup_repo_path = backup_dir.path().join("repo"); + + std::fs::rename(repo_path, &backup_repo_path).map_err(|err| { + format!( + "failed to move previous curated plugins repo out of the way at {}: {err}", + repo_path.display() + ) + })?; + + if let Err(err) = std::fs::rename(staged_repo_path, repo_path) { + let rollback_result = std::fs::rename(&backup_repo_path, repo_path); + return match rollback_result { + Ok(()) => Err(format!( + "failed to activate new curated plugins repo at {}: {err}", + repo_path.display() + )), + Err(rollback_err) => { + let backup_path = backup_dir.keep().join("repo"); + Err(format!( + "failed to activate new curated plugins repo at {}: {err}; failed to restore previous repo (left at {}): {rollback_err}", + repo_path.display(), + backup_path.display() + )) + } + }; + } + } else { + std::fs::rename(staged_repo_path, repo_path).map_err(|err| { + format!( + "failed to activate curated plugins repo at {}: {err}", + repo_path.display() + ) + })?; + } + + Ok(()) +} + +fn write_curated_plugins_sha(sha_path: &Path, remote_sha: &str) -> Result<(), String> { + if let Some(parent) = sha_path.parent() { + std::fs::create_dir_all(parent).map_err(|err| { + format!( + "failed to create curated plugins sha directory {}: {err}", + parent.display() + ) + })?; + } + std::fs::write(sha_path, format!("{remote_sha}\n")).map_err(|err| { + format!( + "failed to write curated plugins sha file {}: {err}", + sha_path.display() + ) + }) +} + +fn read_local_git_or_sha_file( + repo_path: &Path, + sha_path: &Path, + git_binary: &str, +) -> Option { + if repo_path.join(".git").is_dir() + && let Ok(sha) = git_head_sha(repo_path, git_binary) + { + return Some(sha); + } + + read_sha_file(sha_path) +} + +fn git_ls_remote_head_sha(git_binary: &str) -> Result { + let output = run_git_command_with_timeout( + Command::new(git_binary) + .env("GIT_OPTIONAL_LOCKS", "0") + .arg("ls-remote") + .arg("https://github.com/openai/plugins.git") + .arg("HEAD"), + "git ls-remote curated plugins repo", + CURATED_PLUGINS_GIT_TIMEOUT, + )?; + ensure_git_success(&output, "git ls-remote curated plugins repo")?; + + let stdout = String::from_utf8_lossy(&output.stdout); + let Some(first_line) = stdout.lines().next() else { + return Err("git ls-remote returned empty output for curated plugins repo".to_string()); + }; + let Some((sha, _)) = first_line.split_once('\t') else { + return Err(format!( + "unexpected git ls-remote output for curated plugins repo: {first_line}" + )); + }; + if sha.is_empty() { + return Err("git ls-remote returned empty sha for curated plugins repo".to_string()); + } + Ok(sha.to_string()) +} + +fn git_head_sha(repo_path: &Path, git_binary: &str) -> Result { + let output = Command::new(git_binary) + .env("GIT_OPTIONAL_LOCKS", "0") + .arg("-C") + .arg(repo_path) + .arg("rev-parse") + .arg("HEAD") + .output() + .map_err(|err| { + format!( + "failed to run git rev-parse HEAD in {}: {err}", + repo_path.display() + ) + })?; + ensure_git_success(&output, "git rev-parse HEAD")?; + + let sha = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if sha.is_empty() { + return Err(format!( + "git rev-parse HEAD returned empty output in {}", + repo_path.display() + )); + } + Ok(sha) +} + +fn run_git_command_with_timeout( + command: &mut Command, + context: &str, + timeout: Duration, +) -> Result { + let mut child = command + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|err| format!("failed to run {context}: {err}"))?; + + let start = std::time::Instant::now(); + loop { + match child.try_wait() { + Ok(Some(_)) => { + return child + .wait_with_output() + .map_err(|err| format!("failed to wait for {context}: {err}")); + } + Ok(None) => {} + Err(err) => return Err(format!("failed to poll {context}: {err}")), + } + + if start.elapsed() >= timeout { + match child.try_wait() { + Ok(Some(_)) => { + return child + .wait_with_output() + .map_err(|err| format!("failed to wait for {context}: {err}")); + } + Ok(None) => {} + Err(err) => return Err(format!("failed to poll {context}: {err}")), + } + + let _ = child.kill(); + let output = child + .wait_with_output() + .map_err(|err| format!("failed to wait for {context} after timeout: {err}"))?; + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + return if stderr.is_empty() { + Err(format!("{context} timed out after {}s", timeout.as_secs())) + } else { + Err(format!( + "{context} timed out after {}s: {stderr}", + timeout.as_secs() + )) + }; + } + + std::thread::sleep(Duration::from_millis(100)); + } +} + +fn ensure_git_success(output: &Output, context: &str) -> Result<(), String> { + if output.status.success() { + return Ok(()); + } + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + if stderr.is_empty() { + Err(format!("{context} failed with status {}", output.status)) + } else { + Err(format!( + "{context} failed with status {}: {stderr}", + output.status + )) + } +} + +async fn fetch_curated_repo_remote_sha(api_base_url: &str) -> Result { + let api_base_url = api_base_url.trim_end_matches('/'); + let repo_url = format!("{api_base_url}/repos/{OPENAI_PLUGINS_OWNER}/{OPENAI_PLUGINS_REPO}"); + let client = build_reqwest_client(); + let repo_body = fetch_github_text(&client, &repo_url, "get curated plugins repository").await?; + let repo_summary: GitHubRepositorySummary = + serde_json::from_str(&repo_body).map_err(|err| { + format!("failed to parse curated plugins repository response from {repo_url}: {err}") + })?; + if repo_summary.default_branch.is_empty() { + return Err(format!( + "curated plugins repository response from {repo_url} did not include a default branch" + )); + } + + let git_ref_url = format!("{repo_url}/git/ref/heads/{}", repo_summary.default_branch); + let git_ref_body = + fetch_github_text(&client, &git_ref_url, "get curated plugins HEAD ref").await?; + let git_ref: GitHubGitRefSummary = serde_json::from_str(&git_ref_body).map_err(|err| { + format!("failed to parse curated plugins ref response from {git_ref_url}: {err}") + })?; + if git_ref.object.sha.is_empty() { + return Err(format!( + "curated plugins ref response from {git_ref_url} did not include a HEAD sha" + )); + } + + Ok(git_ref.object.sha) +} + +async fn fetch_curated_repo_zipball( + api_base_url: &str, + remote_sha: &str, +) -> Result, String> { + let api_base_url = api_base_url.trim_end_matches('/'); + let repo_url = format!("{api_base_url}/repos/{OPENAI_PLUGINS_OWNER}/{OPENAI_PLUGINS_REPO}"); + let zipball_url = format!("{repo_url}/zipball/{remote_sha}"); + let client = build_reqwest_client(); + fetch_github_bytes(&client, &zipball_url, "download curated plugins archive").await +} + +async fn fetch_curated_repo_backup_archive_zip( + backup_archive_api_url: &str, +) -> Result, String> { + let client = build_reqwest_client(); + let export_body = fetch_public_text( + &client, + backup_archive_api_url, + "get curated plugins export archive metadata", + ) + .await?; + let export_response: CuratedPluginsBackupArchiveResponse = serde_json::from_str(&export_body) + .map_err(|err| { + format!( + "failed to parse curated plugins backup archive response from {backup_archive_api_url}: {err}" + ) + })?; + if export_response.download_url.is_empty() { + return Err(format!( + "curated plugins backup archive response from {backup_archive_api_url} did not include a download URL" + )); + } + + fetch_public_bytes( + &client, + &export_response.download_url, + "download curated plugins export archive", + ) + .await +} + +fn read_extracted_backup_archive_git_sha(repo_path: &Path) -> Result, String> { + let git_dir = repo_path.join(".git"); + if !git_dir.is_dir() { + return Ok(None); + } + + let head_path = git_dir.join("HEAD"); + let head = std::fs::read_to_string(&head_path).map_err(|err| { + format!( + "failed to read curated plugins backup archive git HEAD {}: {err}", + head_path.display() + ) + })?; + let head = head.trim(); + if head.is_empty() { + return Err(format!( + "curated plugins backup archive git HEAD is empty at {}", + head_path.display() + )); + } + + if let Some(reference) = head.strip_prefix("ref: ") { + let reference = validate_backup_archive_git_ref(reference.trim())?; + return read_git_ref_sha(&git_dir, reference).map(Some); + } + + Ok(Some(head.to_string())) +} + +fn validate_backup_archive_git_ref(reference: &str) -> Result<&str, String> { + if !reference.starts_with("refs/") { + return Err(format!( + "curated plugins backup archive git ref must stay under refs/: {reference}" + )); + } + + let path = Path::new(reference); + if path.is_absolute() { + return Err(format!( + "curated plugins backup archive git ref must be relative: {reference}" + )); + } + + for component in path.components() { + match component { + std::path::Component::Normal(_) => {} + _ => { + return Err(format!( + "curated plugins backup archive git ref contains invalid path components: {reference}" + )); + } + } + } + + Ok(reference) +} + +fn read_git_ref_sha(git_dir: &Path, reference: &str) -> Result { + let ref_path = git_dir.join(reference); + if let Ok(sha) = std::fs::read_to_string(&ref_path) { + let sha = sha.trim(); + if sha.is_empty() { + return Err(format!( + "curated plugins backup archive git ref {reference} is empty at {}", + ref_path.display() + )); + } + return Ok(sha.to_string()); + } + + let packed_refs_path = git_dir.join("packed-refs"); + if let Ok(packed_refs) = std::fs::read_to_string(&packed_refs_path) + && let Some(sha) = packed_refs.lines().find_map(|line| { + let trimmed = line.trim(); + if trimmed.is_empty() || trimmed.starts_with('#') || trimmed.starts_with('^') { + return None; + } + let (sha, candidate_ref) = trimmed.split_once(' ')?; + (candidate_ref == reference).then_some(sha.to_string()) + }) + { + return Ok(sha); + } + + Err(format!( + "failed to resolve curated plugins backup archive git ref {reference} from {}", + git_dir.display() + )) +} + +async fn fetch_github_text(client: &Client, url: &str, context: &str) -> Result { + let response = github_request(client, url) + .send() + .await + .map_err(|err| format!("failed to {context} from {url}: {err}"))?; + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + if !status.is_success() { + return Err(format!( + "{context} from {url} failed with status {status}: {body}" + )); + } + Ok(body) +} + +async fn fetch_github_bytes(client: &Client, url: &str, context: &str) -> Result, String> { + let response = github_request(client, url) + .send() + .await + .map_err(|err| format!("failed to {context} from {url}: {err}"))?; + let status = response.status(); + let body = response + .bytes() + .await + .map_err(|err| format!("failed to read {context} response from {url}: {err}"))?; + if !status.is_success() { + let body_text = String::from_utf8_lossy(&body); + return Err(format!( + "{context} from {url} failed with status {status}: {body_text}" + )); + } + Ok(body.to_vec()) +} + +async fn fetch_public_text(client: &Client, url: &str, context: &str) -> Result { + let response = client + .get(url) + .timeout(CURATED_PLUGINS_BACKUP_ARCHIVE_TIMEOUT) + .send() + .await + .map_err(|err| format!("failed to {context} from {url}: {err}"))?; + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + if !status.is_success() { + return Err(format!( + "{context} from {url} failed with status {status}: {body}" + )); + } + Ok(body) +} + +async fn fetch_public_bytes(client: &Client, url: &str, context: &str) -> Result, String> { + let response = client + .get(url) + .timeout(CURATED_PLUGINS_BACKUP_ARCHIVE_TIMEOUT) + .send() + .await + .map_err(|err| format!("failed to {context} from {url}: {err}"))?; + let status = response.status(); + let body = response + .bytes() + .await + .map_err(|err| format!("failed to read {context} response from {url}: {err}"))?; + if !status.is_success() { + let body_text = String::from_utf8_lossy(&body); + return Err(format!( + "{context} from {url} failed with status {status}: {body_text}" + )); + } + Ok(body.to_vec()) +} + +fn github_request(client: &Client, url: &str) -> reqwest::RequestBuilder { + client + .get(url) + .timeout(CURATED_PLUGINS_HTTP_TIMEOUT) + .header("accept", GITHUB_API_ACCEPT_HEADER) + .header("x-github-api-version", GITHUB_API_VERSION_HEADER) +} + +fn read_sha_file(sha_path: &Path) -> Option { + std::fs::read_to_string(sha_path) + .ok() + .map(|sha| sha.trim().to_string()) + .filter(|sha| !sha.is_empty()) +} + +fn extract_zipball_to_dir(bytes: &[u8], destination: &Path) -> Result<(), String> { + std::fs::create_dir_all(destination).map_err(|err| { + format!( + "failed to create curated plugins extraction directory {}: {err}", + destination.display() + ) + })?; + + let cursor = std::io::Cursor::new(bytes); + let mut archive = ZipArchive::new(cursor) + .map_err(|err| format!("failed to open curated plugins zip archive: {err}"))?; + + for index in 0..archive.len() { + let mut entry = archive + .by_index(index) + .map_err(|err| format!("failed to read curated plugins zip entry: {err}"))?; + let Some(relative_path) = entry.enclosed_name() else { + return Err(format!( + "curated plugins zip entry `{}` escapes extraction root", + entry.name() + )); + }; + + let mut components = relative_path.components(); + let Some(std::path::Component::Normal(_)) = components.next() else { + continue; + }; + + let output_relative = components.fold(PathBuf::new(), |mut path, component| { + if let std::path::Component::Normal(segment) = component { + path.push(segment); + } + path + }); + if output_relative.as_os_str().is_empty() { + continue; + } + + let output_path = destination.join(&output_relative); + if entry.is_dir() { + std::fs::create_dir_all(&output_path).map_err(|err| { + format!( + "failed to create curated plugins directory {}: {err}", + output_path.display() + ) + })?; + continue; + } + + if let Some(parent) = output_path.parent() { + std::fs::create_dir_all(parent).map_err(|err| { + format!( + "failed to create curated plugins directory {}: {err}", + parent.display() + ) + })?; + } + let mut output = std::fs::File::create(&output_path).map_err(|err| { + format!( + "failed to create curated plugins file {}: {err}", + output_path.display() + ) + })?; + std::io::copy(&mut entry, &mut output).map_err(|err| { + format!( + "failed to write curated plugins file {}: {err}", + output_path.display() + ) + })?; + apply_zip_permissions(&entry, &output_path)?; + } + + Ok(()) +} + +#[cfg(unix)] +fn apply_zip_permissions(entry: &zip::read::ZipFile<'_>, output_path: &Path) -> Result<(), String> { + use std::os::unix::fs::PermissionsExt; + + let Some(mode) = entry.unix_mode() else { + return Ok(()); + }; + std::fs::set_permissions(output_path, std::fs::Permissions::from_mode(mode)).map_err(|err| { + format!( + "failed to set permissions on curated plugins file {}: {err}", + output_path.display() + ) + }) +} + +#[cfg(not(unix))] +fn apply_zip_permissions( + _entry: &zip::read::ZipFile<'_>, + _output_path: &Path, +) -> Result<(), String> { + Ok(()) +} + +#[cfg(test)] +#[path = "startup_sync_tests.rs"] +mod tests; diff --git a/codex-rs/core-plugins/src/startup_sync_tests.rs b/codex-rs/core-plugins/src/startup_sync_tests.rs new file mode 100644 index 0000000000..a9388e3fce --- /dev/null +++ b/codex-rs/core-plugins/src/startup_sync_tests.rs @@ -0,0 +1,769 @@ +use super::*; +use pretty_assertions::assert_eq; +use std::io::Write; +use std::path::Path; +use std::path::PathBuf; +use tempfile::tempdir; +use wiremock::Mock; +use wiremock::MockServer; +use wiremock::ResponseTemplate; +use wiremock::matchers::method; +use wiremock::matchers::path; +use zip::ZipWriter; +use zip::write::SimpleFileOptions; + +const TEST_CURATED_PLUGIN_SHA: &str = "0123456789abcdef0123456789abcdef01234567"; + +fn write_file(path: &Path, contents: &str) { + std::fs::create_dir_all(path.parent().expect("file should have a parent")).unwrap(); + std::fs::write(path, contents).unwrap(); +} + +fn write_curated_plugin(root: &Path, plugin_name: &str) { + let plugin_root = root.join("plugins").join(plugin_name); + write_file( + &plugin_root.join(".codex-plugin/plugin.json"), + &format!(r#"{{"name":"{plugin_name}"}}"#), + ); +} + +fn write_openai_curated_marketplace(root: &Path, plugin_names: &[&str]) { + let plugins = plugin_names + .iter() + .map(|plugin_name| { + format!( + r#"{{ + "name": "{plugin_name}", + "source": {{ + "source": "local", + "path": "./plugins/{plugin_name}" + }} + }}"# + ) + }) + .collect::>() + .join(",\n"); + write_file( + &root.join(".agents/plugins/marketplace.json"), + &format!( + r#"{{ + "name": "openai-curated", + "plugins": [ +{plugins} + ] +}}"# + ), + ); + for plugin_name in plugin_names { + write_curated_plugin(root, plugin_name); + } +} + +fn write_curated_plugin_sha(codex_home: &Path) { + write_file( + &codex_home.join(".tmp/plugins.sha"), + &format!("{TEST_CURATED_PLUGIN_SHA}\n"), + ); +} + +fn has_plugins_clone_dirs(codex_home: &Path) -> bool { + let Ok(entries) = std::fs::read_dir(codex_home.join(".tmp")) else { + return false; + }; + + entries.flatten().any(|entry| { + let path = entry.path(); + path.is_dir() + && path + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with("plugins-clone-")) + }) +} + +#[cfg(unix)] +fn write_executable_script(path: &Path, contents: &str) { + #[cfg(unix)] + use std::os::unix::fs::PermissionsExt; + + std::fs::write(path, contents).expect("write script"); + #[cfg(unix)] + { + let mut permissions = std::fs::metadata(path).expect("metadata").permissions(); + permissions.set_mode(0o755); + std::fs::set_permissions(path, permissions).expect("chmod"); + } +} + +async fn mount_github_repo_and_ref(server: &MockServer, sha: &str) { + Mock::given(method("GET")) + .and(path("/repos/openai/plugins")) + .respond_with(ResponseTemplate::new(200).set_body_string(r#"{"default_branch":"main"}"#)) + .mount(server) + .await; + Mock::given(method("GET")) + .and(path("/repos/openai/plugins/git/ref/heads/main")) + .respond_with( + ResponseTemplate::new(200) + .set_body_string(format!(r#"{{"object":{{"sha":"{sha}"}}}}"#)), + ) + .mount(server) + .await; +} + +async fn mount_github_zipball(server: &MockServer, sha: &str, bytes: Vec) { + Mock::given(method("GET")) + .and(path(format!("/repos/openai/plugins/zipball/{sha}"))) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", "application/zip") + .set_body_bytes(bytes), + ) + .mount(server) + .await; +} + +async fn mount_export_archive(server: &MockServer, bytes: Vec) -> String { + let export_api_url = format!("{}/backend-api/plugins/export/curated", server.uri()); + Mock::given(method("GET")) + .and(path("/backend-api/plugins/export/curated")) + .respond_with(ResponseTemplate::new(200).set_body_string(format!( + r#"{{"download_url":"{}/files/curated-plugins.zip"}}"#, + server.uri() + ))) + .mount(server) + .await; + Mock::given(method("GET")) + .and(path("/files/curated-plugins.zip")) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", "application/zip") + .set_body_bytes(bytes), + ) + .mount(server) + .await; + export_api_url +} + +async fn run_sync_with_transport_overrides( + codex_home: PathBuf, + git_binary: impl Into, + api_base_url: impl Into, + backup_archive_api_url: impl Into, +) -> Result { + let git_binary = git_binary.into(); + let api_base_url = api_base_url.into(); + let backup_archive_api_url = backup_archive_api_url.into(); + tokio::task::spawn_blocking(move || { + sync_openai_plugins_repo_with_transport_overrides( + codex_home.as_path(), + &git_binary, + &api_base_url, + &backup_archive_api_url, + ) + }) + .await + .expect("sync task should join") +} + +async fn run_http_sync( + codex_home: PathBuf, + api_base_url: impl Into, +) -> Result { + let api_base_url = api_base_url.into(); + tokio::task::spawn_blocking(move || { + sync_openai_plugins_repo_via_http(codex_home.as_path(), &api_base_url) + }) + .await + .expect("sync task should join") +} + +fn assert_curated_gmail_repo(repo_path: &Path) { + assert!(repo_path.join(".agents/plugins/marketplace.json").is_file()); + assert!( + repo_path + .join("plugins/gmail/.codex-plugin/plugin.json") + .is_file() + ); +} + +#[test] +fn curated_plugins_repo_path_uses_codex_home_tmp_dir() { + let tmp = tempdir().expect("tempdir"); + assert_eq!( + curated_plugins_repo_path(tmp.path()), + tmp.path().join(".tmp/plugins") + ); +} + +#[test] +fn read_curated_plugins_sha_reads_trimmed_sha_file() { + let tmp = tempdir().expect("tempdir"); + std::fs::create_dir_all(tmp.path().join(".tmp")).expect("create tmp"); + std::fs::write(tmp.path().join(".tmp/plugins.sha"), "abc123\n").expect("write sha"); + + assert_eq!( + read_curated_plugins_sha(tmp.path()).as_deref(), + Some("abc123") + ); +} + +#[cfg(unix)] +#[test] +fn remove_stale_curated_repo_temp_dirs_removes_only_matching_directories() { + use std::os::unix::ffi::OsStrExt; + use std::time::SystemTime; + + fn set_dir_mtime(path: &Path, age: Duration) -> Result<(), Box> { + let now = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH)?; + let modified_at = now.saturating_sub(age); + let tv_sec = i64::try_from(modified_at.as_secs())?; + let ts = libc::timespec { tv_sec, tv_nsec: 0 }; + let times = [ts, ts]; + let c_path = std::ffi::CString::new(path.as_os_str().as_bytes())?; + let result = unsafe { libc::utimensat(libc::AT_FDCWD, c_path.as_ptr(), times.as_ptr(), 0) }; + if result != 0 { + return Err(std::io::Error::last_os_error().into()); + } + Ok(()) + } + + let tmp = tempdir().expect("tempdir"); + let parent = tmp.path().join(".tmp"); + let stale_clone_dir = parent.join("plugins-clone-stale"); + let fresh_clone_dir = parent.join("plugins-clone-fresh"); + let unrelated_dir = parent.join("plugins-cache"); + + std::fs::create_dir_all(&stale_clone_dir).expect("create stale clone dir"); + std::fs::create_dir_all(&fresh_clone_dir).expect("create fresh clone dir"); + std::fs::create_dir_all(&unrelated_dir).expect("create unrelated dir"); + set_dir_mtime( + &stale_clone_dir, + CURATED_PLUGINS_STALE_TEMP_DIR_MAX_AGE + Duration::from_secs(60), + ) + .expect("age stale clone dir"); + set_dir_mtime(&fresh_clone_dir, Duration::ZERO).expect("age fresh clone dir"); + + remove_stale_curated_repo_temp_dirs(&parent, CURATED_PLUGINS_STALE_TEMP_DIR_MAX_AGE); + + assert!(!stale_clone_dir.exists()); + assert!(fresh_clone_dir.is_dir()); + assert!(unrelated_dir.is_dir()); +} + +#[cfg(unix)] +#[test] +fn sync_openai_plugins_repo_prefers_git_when_available() { + let tmp = tempdir().expect("tempdir"); + let bin_dir = tempfile::Builder::new() + .prefix("fake-git-") + .tempdir() + .expect("tempdir"); + let git_path = bin_dir.path().join("git"); + let sha = "0123456789abcdef0123456789abcdef01234567"; + + write_executable_script( + &git_path, + &format!( + r#"#!/bin/sh +if [ "$1" = "ls-remote" ]; then + printf '%s\tHEAD\n' "{sha}" + exit 0 +fi +if [ "$1" = "clone" ]; then + dest="$5" + mkdir -p "$dest/.git" "$dest/.agents/plugins" "$dest/plugins/gmail/.codex-plugin" + cat > "$dest/.agents/plugins/marketplace.json" <<'EOF' +{{"name":"openai-curated","plugins":[{{"name":"gmail","source":{{"source":"local","path":"./plugins/gmail"}}}}]}} +EOF + printf '%s\n' '{{"name":"gmail"}}' > "$dest/plugins/gmail/.codex-plugin/plugin.json" + exit 0 +fi +if [ "$1" = "-C" ] && [ "$3" = "rev-parse" ] && [ "$4" = "HEAD" ]; then + printf '%s\n' "{sha}" + exit 0 +fi +echo "unexpected git invocation: $@" >&2 +exit 1 +"# + ), + ); + + let synced_sha = sync_openai_plugins_repo_with_transport_overrides( + tmp.path(), + git_path.to_str().expect("utf8 path"), + "http://127.0.0.1:9", + "http://127.0.0.1:9/backend-api/plugins/export/curated", + ) + .expect("git sync should succeed"); + + assert_eq!(synced_sha, sha); + let repo_path = curated_plugins_repo_path(tmp.path()); + assert!(repo_path.join(".git").is_dir()); + assert_curated_gmail_repo(&repo_path); + assert_eq!(read_curated_plugins_sha(tmp.path()).as_deref(), Some(sha)); +} + +#[cfg(unix)] +#[test] +fn sync_openai_plugins_repo_via_git_succeeds_with_local_rewritten_remote() { + let tmp = tempdir().expect("tempdir"); + let repo_root = tempfile::Builder::new() + .prefix("curated-repo-success-") + .tempdir() + .expect("tempdir"); + let work_repo = repo_root.path().join("work/plugins"); + let remote_repo = repo_root.path().join("remotes/openai/plugins.git"); + std::fs::create_dir_all(work_repo.join(".agents/plugins")).expect("create marketplace dir"); + std::fs::create_dir_all(work_repo.join("plugins/gmail/.codex-plugin")) + .expect("create plugin dir"); + std::fs::write( + work_repo.join(".agents/plugins/marketplace.json"), + r#"{"name":"openai-curated","plugins":[{"name":"gmail","source":{"source":"local","path":"./plugins/gmail"}}]}"#, + ) + .expect("write marketplace"); + std::fs::write( + work_repo.join("plugins/gmail/.codex-plugin/plugin.json"), + r#"{"name":"gmail"}"#, + ) + .expect("write plugin manifest"); + + let init_status = Command::new("git") + .arg("-C") + .arg(&work_repo) + .arg("init") + .status() + .expect("run git init"); + assert!(init_status.success()); + + let add_status = Command::new("git") + .arg("-C") + .arg(&work_repo) + .arg("add") + .arg(".") + .status() + .expect("run git add"); + assert!(add_status.success()); + + let commit_status = Command::new("git") + .arg("-C") + .arg(&work_repo) + .arg("-c") + .arg("user.name=Codex Test") + .arg("-c") + .arg("user.email=codex@example.com") + .arg("commit") + .arg("-m") + .arg("init") + .status() + .expect("run git commit"); + assert!(commit_status.success()); + + std::fs::create_dir_all(remote_repo.parent().expect("remote parent")) + .expect("create remote parent"); + let clone_status = Command::new("git") + .arg("clone") + .arg("--bare") + .arg(&work_repo) + .arg(&remote_repo) + .status() + .expect("run git clone --bare"); + assert!(clone_status.success()); + + let sha_output = Command::new("git") + .arg("-C") + .arg(&work_repo) + .arg("rev-parse") + .arg("HEAD") + .output() + .expect("run git rev-parse"); + assert!(sha_output.status.success()); + let sha = String::from_utf8_lossy(&sha_output.stdout) + .trim() + .to_string(); + + let git_config_path = repo_root.path().join("git-rewrite.conf"); + std::fs::write( + &git_config_path, + format!( + "[url \"file://{}/\"]\n insteadOf = https://github.com/\n", + repo_root.path().join("remotes").display() + ), + ) + .expect("write git config"); + + let bin_dir = tempfile::Builder::new() + .prefix("git-rewrite-wrapper-") + .tempdir() + .expect("tempdir"); + let git_wrapper = bin_dir.path().join("git"); + write_executable_script( + &git_wrapper, + &format!( + "#!/bin/sh\nGIT_CONFIG_GLOBAL='{}' exec git \"$@\"\n", + git_config_path.display() + ), + ); + + let synced_sha = + sync_openai_plugins_repo_via_git(tmp.path(), git_wrapper.to_str().expect("utf8 path")) + .expect("git sync should succeed"); + + assert_eq!(synced_sha, sha); + assert_curated_gmail_repo(&curated_plugins_repo_path(tmp.path())); + assert_eq!( + read_curated_plugins_sha(tmp.path()).as_deref(), + Some(sha.as_str()) + ); + assert!(!has_plugins_clone_dirs(tmp.path())); +} + +#[tokio::test] +async fn sync_openai_plugins_repo_falls_back_to_http_when_git_is_unavailable() { + let tmp = tempdir().expect("tempdir"); + let server = MockServer::start().await; + let sha = "0123456789abcdef0123456789abcdef01234567"; + + mount_github_repo_and_ref(&server, sha).await; + mount_github_zipball(&server, sha, curated_repo_zipball_bytes(sha)).await; + + let synced_sha = run_sync_with_transport_overrides( + tmp.path().to_path_buf(), + "missing-git-for-test", + server.uri(), + "http://127.0.0.1:9/backend-api/plugins/export/curated", + ) + .await + .expect("fallback sync should succeed"); + + let repo_path = curated_plugins_repo_path(tmp.path()); + assert_eq!(synced_sha, sha); + assert_curated_gmail_repo(&repo_path); + assert_eq!(read_curated_plugins_sha(tmp.path()).as_deref(), Some(sha)); +} + +#[cfg(unix)] +#[tokio::test] +async fn sync_openai_plugins_repo_falls_back_to_http_when_git_sync_fails() { + let tmp = tempdir().expect("tempdir"); + let bin_dir = tempfile::Builder::new() + .prefix("fake-git-fail-") + .tempdir() + .expect("tempdir"); + let git_path = bin_dir.path().join("git"); + let sha = "0123456789abcdef0123456789abcdef01234567"; + + write_executable_script( + &git_path, + r#"#!/bin/sh +echo "simulated git failure" >&2 +exit 1 +"#, + ); + + let server = MockServer::start().await; + mount_github_repo_and_ref(&server, sha).await; + mount_github_zipball(&server, sha, curated_repo_zipball_bytes(sha)).await; + + let synced_sha = run_sync_with_transport_overrides( + tmp.path().to_path_buf(), + git_path.to_str().expect("utf8 path"), + server.uri(), + "http://127.0.0.1:9/backend-api/plugins/export/curated", + ) + .await + .expect("fallback sync should succeed"); + + let repo_path = curated_plugins_repo_path(tmp.path()); + assert_eq!(synced_sha, sha); + assert_curated_gmail_repo(&repo_path); + assert_eq!(read_curated_plugins_sha(tmp.path()).as_deref(), Some(sha)); +} + +#[cfg(unix)] +#[test] +fn sync_openai_plugins_repo_via_git_cleans_up_staged_dir_on_clone_failure() { + let tmp = tempdir().expect("tempdir"); + let bin_dir = tempfile::Builder::new() + .prefix("fake-git-partial-fail-") + .tempdir() + .expect("tempdir"); + let git_path = bin_dir.path().join("git"); + let sha = "0123456789abcdef0123456789abcdef01234567"; + + write_executable_script( + &git_path, + &format!( + r#"#!/bin/sh +if [ "$1" = "ls-remote" ]; then + printf '%s\tHEAD\n' "{sha}" + exit 0 +fi +if [ "$1" = "clone" ]; then + dest="$5" + mkdir -p "$dest/.git" + echo "fatal: early EOF" >&2 + exit 128 +fi +echo "unexpected git invocation: $@" >&2 +exit 1 +"# + ), + ); + + let err = sync_openai_plugins_repo_via_git(tmp.path(), git_path.to_str().expect("utf8 path")) + .expect_err("git sync should fail"); + + assert!(err.contains("fatal: early EOF")); + assert!(!has_plugins_clone_dirs(tmp.path())); +} + +#[tokio::test] +async fn sync_openai_plugins_repo_via_http_cleans_up_staged_dir_on_extract_failure() { + let tmp = tempdir().expect("tempdir"); + let server = MockServer::start().await; + let sha = "0123456789abcdef0123456789abcdef01234567"; + + mount_github_repo_and_ref(&server, sha).await; + mount_github_zipball(&server, sha, b"not a zip archive".to_vec()).await; + + let err = run_http_sync(tmp.path().to_path_buf(), server.uri()) + .await + .expect_err("http sync should fail"); + + assert!(err.contains("failed to open curated plugins zip archive")); + assert!(!has_plugins_clone_dirs(tmp.path())); +} + +#[tokio::test] +async fn sync_openai_plugins_repo_skips_archive_download_when_sha_matches() { + let tmp = tempdir().expect("tempdir"); + let repo_path = curated_plugins_repo_path(tmp.path()); + std::fs::create_dir_all(repo_path.join(".agents/plugins")).expect("create repo"); + std::fs::write( + repo_path.join(".agents/plugins/marketplace.json"), + r#"{"name":"openai-curated","plugins":[]}"#, + ) + .expect("write marketplace"); + std::fs::create_dir_all(tmp.path().join(".tmp")).expect("create tmp"); + let sha = "fedcba9876543210fedcba9876543210fedcba98"; + std::fs::write(tmp.path().join(".tmp/plugins.sha"), format!("{sha}\n")).expect("write sha"); + + let server = MockServer::start().await; + mount_github_repo_and_ref(&server, sha).await; + + run_sync_with_transport_overrides( + tmp.path().to_path_buf(), + "missing-git-for-test", + server.uri(), + "http://127.0.0.1:9/backend-api/plugins/export/curated", + ) + .await + .expect("sync should succeed"); + + assert_eq!(read_curated_plugins_sha(tmp.path()).as_deref(), Some(sha)); + assert!(repo_path.join(".agents/plugins/marketplace.json").is_file()); +} + +#[tokio::test] +async fn sync_openai_plugins_repo_falls_back_to_export_archive_when_no_snapshot_exists() { + let tmp = tempdir().expect("tempdir"); + let server = MockServer::start().await; + let export_sha = "1111111111111111111111111111111111111111"; + + Mock::given(method("GET")) + .and(path("/repos/openai/plugins")) + .respond_with(ResponseTemplate::new(500).set_body_string("github repo lookup failed")) + .mount(&server) + .await; + let export_api_url = + mount_export_archive(&server, curated_repo_backup_archive_zip_bytes(export_sha)).await; + + let synced_sha = run_sync_with_transport_overrides( + tmp.path().to_path_buf(), + "missing-git-for-test", + server.uri(), + export_api_url, + ) + .await + .expect("export fallback sync should succeed"); + + let repo_path = curated_plugins_repo_path(tmp.path()); + assert_eq!(synced_sha, export_sha); + assert_curated_gmail_repo(&repo_path); + assert_eq!( + read_curated_plugins_sha(tmp.path()).as_deref(), + Some(export_sha) + ); +} + +#[tokio::test] +async fn sync_openai_plugins_repo_skips_export_archive_when_snapshot_exists() { + let tmp = tempdir().expect("tempdir"); + let curated_root = curated_plugins_repo_path(tmp.path()); + write_openai_curated_marketplace(&curated_root, &["linear"]); + write_curated_plugin_sha(tmp.path()); + + let plugin_manifest_path = curated_root.join("plugins/linear/.codex-plugin/plugin.json"); + let original_manifest = + std::fs::read_to_string(&plugin_manifest_path).expect("read existing plugin manifest"); + + let server = MockServer::start().await; + + Mock::given(method("GET")) + .and(path("/repos/openai/plugins")) + .respond_with(ResponseTemplate::new(500).set_body_string("github repo lookup failed")) + .mount(&server) + .await; + let export_api_url = mount_export_archive( + &server, + curated_repo_backup_archive_zip_bytes("2222222222222222222222222222222222222222"), + ) + .await; + + let err = run_sync_with_transport_overrides( + tmp.path().to_path_buf(), + "missing-git-for-test", + server.uri(), + export_api_url, + ) + .await + .expect_err("existing snapshot should suppress export fallback"); + + assert!(err.contains("export archive fallback skipped")); + assert_eq!( + std::fs::read_to_string(&plugin_manifest_path).expect("read plugin manifest after sync"), + original_manifest + ); + assert_eq!( + read_curated_plugins_sha(tmp.path()).as_deref(), + Some(TEST_CURATED_PLUGIN_SHA) + ); +} + +#[test] +fn read_extracted_backup_archive_git_sha_reads_head_ref_from_extracted_repo() { + let tmp = tempdir().expect("tempdir"); + let git_dir = tmp.path().join(".git/refs/heads"); + std::fs::create_dir_all(&git_dir).expect("create git ref dir"); + std::fs::write(tmp.path().join(".git/HEAD"), "ref: refs/heads/main\n").expect("write HEAD"); + std::fs::write( + git_dir.join("main"), + "3333333333333333333333333333333333333333\n", + ) + .expect("write main ref"); + + assert_eq!( + read_extracted_backup_archive_git_sha(tmp.path()) + .expect("read extracted backup archive git sha"), + Some("3333333333333333333333333333333333333333".to_string()) + ); +} + +#[test] +fn read_extracted_backup_archive_git_sha_rejects_non_refs_head_target() { + let tmp = tempdir().expect("tempdir"); + std::fs::create_dir_all(tmp.path().join(".git")).expect("create git dir"); + std::fs::write(tmp.path().join(".git/HEAD"), "ref: HEAD\n").expect("write HEAD"); + + let err = read_extracted_backup_archive_git_sha(tmp.path()) + .expect_err("non-refs target should be rejected"); + + assert!(err.contains("must stay under refs/")); +} + +#[test] +fn read_extracted_backup_archive_git_sha_rejects_path_traversal_ref() { + let tmp = tempdir().expect("tempdir"); + std::fs::create_dir_all(tmp.path().join(".git")).expect("create git dir"); + std::fs::write(tmp.path().join(".git/HEAD"), "ref: refs/heads/../../evil\n") + .expect("write HEAD"); + + let err = read_extracted_backup_archive_git_sha(tmp.path()) + .expect_err("path traversal ref should be rejected"); + + assert!(err.contains("invalid path components")); +} + +fn curated_repo_zipball_bytes(sha: &str) -> Vec { + let cursor = std::io::Cursor::new(Vec::new()); + let mut writer = ZipWriter::new(cursor); + let options = SimpleFileOptions::default(); + let root = format!("openai-plugins-{sha}"); + writer + .start_file(format!("{root}/.agents/plugins/marketplace.json"), options) + .expect("start marketplace entry"); + writer + .write_all( + br#"{ + "name": "openai-curated", + "plugins": [ + { + "name": "gmail", + "source": { + "source": "local", + "path": "./plugins/gmail" + } + } + ] +}"#, + ) + .expect("write marketplace"); + writer + .start_file( + format!("{root}/plugins/gmail/.codex-plugin/plugin.json"), + options, + ) + .expect("start plugin manifest entry"); + writer + .write_all(br#"{"name":"gmail"}"#) + .expect("write plugin manifest"); + + writer.finish().expect("finish zip writer").into_inner() +} + +fn curated_repo_backup_archive_zip_bytes(sha: &str) -> Vec { + let cursor = std::io::Cursor::new(Vec::new()); + let mut writer = ZipWriter::new(cursor); + let options = SimpleFileOptions::default(); + + writer + .start_file("plugins/.git/HEAD", options) + .expect("start HEAD entry"); + writer + .write_all(b"ref: refs/heads/main\n") + .expect("write HEAD"); + writer + .start_file("plugins/.git/refs/heads/main", options) + .expect("start main ref entry"); + writer + .write_all(format!("{sha}\n").as_bytes()) + .expect("write main ref"); + writer + .start_file("plugins/.agents/plugins/marketplace.json", options) + .expect("start marketplace entry"); + writer + .write_all( + br#"{ + "name": "openai-curated", + "plugins": [ + { + "name": "gmail", + "source": { + "source": "local", + "path": "./plugins/gmail" + } + } + ] +}"#, + ) + .expect("write marketplace"); + writer + .start_file("plugins/plugins/gmail/.codex-plugin/plugin.json", options) + .expect("start plugin manifest entry"); + writer + .write_all(br#"{"name":"gmail"}"#) + .expect("write plugin manifest"); + + writer.finish().expect("finish zip writer").into_inner() +} diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index dccd1f9cf0..42deea9684 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -119,7 +119,6 @@ url = { workspace = true } uuid = { workspace = true, features = ["serde", "v4", "v5"] } which = { workspace = true } whoami = { workspace = true } -zip = { workspace = true } [target.'cfg(target_os = "macos")'.dependencies] core-foundation = "0.9" diff --git a/codex-rs/core/src/plugins/discoverable.rs b/codex-rs/core/src/plugins/discoverable.rs index 6492bcb180..8340f50493 100644 --- a/codex-rs/core/src/plugins/discoverable.rs +++ b/codex-rs/core/src/plugins/discoverable.rs @@ -2,12 +2,12 @@ use anyhow::Context; use std::collections::HashSet; use tracing::warn; -use super::OPENAI_BUNDLED_MARKETPLACE_NAME; -use super::OPENAI_CURATED_MARKETPLACE_NAME; use super::PluginCapabilitySummary; use super::PluginsManager; use crate::config::Config; use codex_config::types::ToolSuggestDiscoverableType; +use codex_core_plugins::OPENAI_BUNDLED_MARKETPLACE_NAME; +use codex_core_plugins::OPENAI_CURATED_MARKETPLACE_NAME; use codex_features::Feature; use codex_tools::DiscoverablePluginInfo; diff --git a/codex-rs/core/src/plugins/discoverable_tests.rs b/codex-rs/core/src/plugins/discoverable_tests.rs index c5f8e6927c..fb25b62f0c 100644 --- a/codex-rs/core/src/plugins/discoverable_tests.rs +++ b/codex-rs/core/src/plugins/discoverable_tests.rs @@ -6,6 +6,7 @@ use crate::plugins::test_support::write_curated_plugin_sha; use crate::plugins::test_support::write_file; use crate::plugins::test_support::write_openai_curated_marketplace; use crate::plugins::test_support::write_plugins_feature_config; +use codex_core_plugins::startup_sync::curated_plugins_repo_path; use codex_tools::DiscoverablePluginInfo; use codex_utils_absolute_path::AbsolutePathBuf; use pretty_assertions::assert_eq; @@ -17,7 +18,7 @@ use tracing_test::internal::MockWriter; #[tokio::test] async fn list_tool_suggest_discoverable_plugins_returns_uninstalled_curated_plugins() { let codex_home = tempdir().expect("tempdir should succeed"); - let curated_root = crate::plugins::curated_plugins_repo_path(codex_home.path()); + let curated_root = curated_plugins_repo_path(codex_home.path()); write_openai_curated_marketplace(&curated_root, &["sample", "slack"]); write_plugins_feature_config(codex_home.path()); @@ -102,7 +103,7 @@ discoverables = [{{ type = "plugin", id = "{plugin_id}" }}] #[tokio::test] async fn list_tool_suggest_discoverable_plugins_ignores_missing_allowlisted_plugin() { let codex_home = tempdir().expect("tempdir should succeed"); - let curated_root = crate::plugins::curated_plugins_repo_path(codex_home.path()); + let curated_root = curated_plugins_repo_path(codex_home.path()); write_openai_curated_marketplace(&curated_root, &["slack"]); let marketplace_name = TOOL_SUGGEST_DISCOVERABLE_PLUGIN_ALLOWLIST .iter() @@ -153,7 +154,7 @@ source = "/tmp/{marketplace_name}" #[tokio::test] async fn list_tool_suggest_discoverable_plugins_returns_empty_when_plugins_feature_disabled() { let codex_home = tempdir().expect("tempdir should succeed"); - let curated_root = crate::plugins::curated_plugins_repo_path(codex_home.path()); + let curated_root = curated_plugins_repo_path(codex_home.path()); write_openai_curated_marketplace(&curated_root, &["slack"]); write_file( &codex_home.path().join(crate::config::CONFIG_TOML_FILE), @@ -173,7 +174,7 @@ plugins = false #[tokio::test] async fn list_tool_suggest_discoverable_plugins_normalizes_description() { let codex_home = tempdir().expect("tempdir should succeed"); - let curated_root = crate::plugins::curated_plugins_repo_path(codex_home.path()); + let curated_root = curated_plugins_repo_path(codex_home.path()); write_openai_curated_marketplace(&curated_root, &["slack"]); write_plugins_feature_config(codex_home.path()); write_file( @@ -205,7 +206,7 @@ async fn list_tool_suggest_discoverable_plugins_normalizes_description() { #[tokio::test] async fn list_tool_suggest_discoverable_plugins_omits_installed_curated_plugins() { let codex_home = tempdir().expect("tempdir should succeed"); - let curated_root = crate::plugins::curated_plugins_repo_path(codex_home.path()); + let curated_root = curated_plugins_repo_path(codex_home.path()); write_openai_curated_marketplace(&curated_root, &["slack"]); write_curated_plugin_sha(codex_home.path()); write_plugins_feature_config(codex_home.path()); @@ -232,7 +233,7 @@ async fn list_tool_suggest_discoverable_plugins_omits_installed_curated_plugins( #[tokio::test] async fn list_tool_suggest_discoverable_plugins_includes_configured_plugin_ids() { let codex_home = tempdir().expect("tempdir should succeed"); - let curated_root = crate::plugins::curated_plugins_repo_path(codex_home.path()); + let curated_root = curated_plugins_repo_path(codex_home.path()); write_openai_curated_marketplace(&curated_root, &["sample"]); write_file( &codex_home.path().join(crate::config::CONFIG_TOML_FILE), @@ -267,7 +268,7 @@ discoverables = [{ type = "plugin", id = "sample@openai-curated" }] #[tokio::test] async fn list_tool_suggest_discoverable_plugins_does_not_reload_marketplace_per_plugin() { let codex_home = tempdir().expect("tempdir should succeed"); - let curated_root = crate::plugins::curated_plugins_repo_path(codex_home.path()); + let curated_root = curated_plugins_repo_path(codex_home.path()); write_openai_curated_marketplace( &curated_root, &["slack", "build-ios-apps", "life-science-research"], diff --git a/codex-rs/core/src/plugins/manager.rs b/codex-rs/core/src/plugins/manager.rs index 7d9b426b26..d47f2c35b8 100644 --- a/codex-rs/core/src/plugins/manager.rs +++ b/codex-rs/core/src/plugins/manager.rs @@ -1,9 +1,5 @@ use super::PluginLoadOutcome; -use super::curated_plugins_repo_path; -use super::installed_marketplaces::installed_marketplace_roots_from_config; -use super::read_curated_plugins_sha; use super::startup_sync::start_startup_remote_plugin_sync_once; -use super::sync_openai_plugins_repo; use crate::SkillMetadata; use crate::config::Config; use crate::config::edit::ConfigEdit; @@ -11,6 +7,8 @@ use crate::config::edit::ConfigEditsBuilder; use crate::config_loader::ConfigLayerStack; use codex_analytics::AnalyticsEventsClient; use codex_config::types::PluginConfig; +use codex_core_plugins::OPENAI_CURATED_MARKETPLACE_NAME; +use codex_core_plugins::installed_marketplaces::installed_marketplace_roots_from_layer_stack; use codex_core_plugins::loader::configured_curated_plugin_ids_from_codex_home; use codex_core_plugins::loader::installed_plugin_telemetry_metadata; use codex_core_plugins::loader::load_plugin_apps; @@ -44,6 +42,9 @@ use codex_core_plugins::marketplace_upgrade::upgrade_configured_git_marketplaces use codex_core_plugins::remote::RemotePluginServiceConfig; use codex_core_plugins::remote_legacy::RemotePluginFetchError; use codex_core_plugins::remote_legacy::RemotePluginMutationError; +use codex_core_plugins::startup_sync::curated_plugins_repo_path; +use codex_core_plugins::startup_sync::read_curated_plugins_sha; +use codex_core_plugins::startup_sync::sync_openai_plugins_repo; use codex_core_plugins::store::PluginInstallResult as StorePluginInstallResult; use codex_core_plugins::store::PluginStore; use codex_core_plugins::store::PluginStoreError; @@ -70,8 +71,6 @@ use toml_edit::value; use tracing::info; use tracing::warn; -pub const OPENAI_CURATED_MARKETPLACE_NAME: &str = "openai-curated"; -pub const OPENAI_BUNDLED_MARKETPLACE_NAME: &str = "openai-bundled"; static CURATED_REPO_SYNC_STARTED: AtomicBool = AtomicBool::new(false); const FEATURED_PLUGIN_IDS_CACHE_TTL: std::time::Duration = std::time::Duration::from_secs(60 * 60 * 3); @@ -1466,8 +1465,8 @@ impl PluginsManager { // Treat the curated catalog as an extra marketplace root so plugin listing can surface it // without requiring every caller to know where it is stored. let mut roots = additional_roots.to_vec(); - roots.extend(installed_marketplace_roots_from_config( - config, + roots.extend(installed_marketplace_roots_from_layer_stack( + &config.config_layer_stack, self.codex_home.as_path(), )); let curated_repo_root = curated_plugins_repo_path(self.codex_home.as_path()); diff --git a/codex-rs/core/src/plugins/manager_tests.rs b/codex-rs/core/src/plugins/manager_tests.rs index c455b08877..8f8efdd713 100644 --- a/codex-rs/core/src/plugins/manager_tests.rs +++ b/codex-rs/core/src/plugins/manager_tests.rs @@ -7,7 +7,6 @@ use crate::config_loader::ConfigRequirements; use crate::config_loader::ConfigRequirementsToml; use crate::plugins::LoadedPlugin; use crate::plugins::PluginLoadOutcome; -use crate::plugins::marketplace_install_root; use crate::plugins::test_support::TEST_CURATED_PLUGIN_SHA; use crate::plugins::test_support::write_curated_plugin_sha_with as write_curated_plugin_sha; use crate::plugins::test_support::write_file; @@ -15,9 +14,11 @@ use crate::plugins::test_support::write_openai_curated_marketplace; use codex_app_server_protocol::ConfigLayerSource; use codex_config::McpServerConfig; use codex_config::types::McpServerTransportConfig; +use codex_core_plugins::installed_marketplaces::marketplace_install_root; use codex_core_plugins::loader::refresh_non_curated_plugin_cache; use codex_core_plugins::loader::refresh_non_curated_plugin_cache_force_reinstall; use codex_core_plugins::marketplace::MarketplacePluginInstallPolicy; +use codex_core_plugins::startup_sync::curated_plugins_repo_path; use codex_login::CodexAuth; use codex_protocol::protocol::Product; use codex_utils_absolute_path::test_support::PathBufExt; diff --git a/codex-rs/core/src/plugins/mod.rs b/codex-rs/core/src/plugins/mod.rs index a8fbcdf98e..ff3183557a 100644 --- a/codex-rs/core/src/plugins/mod.rs +++ b/codex-rs/core/src/plugins/mod.rs @@ -2,10 +2,7 @@ use codex_config::types::McpServerConfig; mod discoverable; mod injection; -mod installed_marketplaces; mod manager; -mod marketplace_add; -mod marketplace_remove; mod mentions; mod render; mod startup_sync; @@ -27,13 +24,9 @@ pub type PluginLoadOutcome = codex_plugin::PluginLoadOutcome; pub(crate) use discoverable::list_tool_suggest_discoverable_plugins; pub(crate) use injection::build_plugin_injections; -pub use installed_marketplaces::INSTALLED_MARKETPLACES_DIR; -pub use installed_marketplaces::marketplace_install_root; pub use manager::ConfiguredMarketplace; pub use manager::ConfiguredMarketplaceListOutcome; pub use manager::ConfiguredMarketplacePlugin; -pub use manager::OPENAI_BUNDLED_MARKETPLACE_NAME; -pub use manager::OPENAI_CURATED_MARKETPLACE_NAME; pub use manager::PluginDetail; pub use manager::PluginDetailsUnavailableReason; pub use manager::PluginInstallError; @@ -45,19 +38,7 @@ pub use manager::PluginRemoteSyncError; pub use manager::PluginUninstallError; pub use manager::PluginsManager; pub use manager::RemotePluginSyncResult; -pub use marketplace_add::MarketplaceAddError; -pub use marketplace_add::MarketplaceAddOutcome; -pub use marketplace_add::MarketplaceAddRequest; -pub use marketplace_add::add_marketplace; -pub use marketplace_add::is_local_marketplace_source; -pub use marketplace_remove::MarketplaceRemoveError; -pub use marketplace_remove::MarketplaceRemoveOutcome; -pub use marketplace_remove::MarketplaceRemoveRequest; -pub use marketplace_remove::remove_marketplace; pub(crate) use render::render_explicit_plugin_instructions; -pub(crate) use startup_sync::curated_plugins_repo_path; -pub(crate) use startup_sync::read_curated_plugins_sha; -pub(crate) use startup_sync::sync_openai_plugins_repo; pub(crate) use mentions::build_connector_slug_counts; pub(crate) use mentions::build_skill_name_counts; diff --git a/codex-rs/core/src/plugins/startup_sync.rs b/codex-rs/core/src/plugins/startup_sync.rs index 5bca32de13..31cf4c75e2 100644 --- a/codex-rs/core/src/plugins/startup_sync.rs +++ b/codex-rs/core/src/plugins/startup_sync.rs @@ -1,235 +1,19 @@ use std::path::Path; use std::path::PathBuf; -use std::process::Command; -use std::process::Output; -use std::process::Stdio; use std::sync::Arc; use std::time::Duration; -use codex_otel::CURATED_PLUGINS_STARTUP_SYNC_FINAL_METRIC; -use codex_otel::CURATED_PLUGINS_STARTUP_SYNC_METRIC; -use reqwest::Client; -use serde::Deserialize; -use tempfile::TempDir; +use crate::config::Config; +use crate::plugins::PluginsManager; +use codex_core_plugins::startup_sync::has_local_curated_plugins_snapshot; +use codex_login::AuthManager; use tracing::info; use tracing::warn; -use zip::ZipArchive; -use crate::config::Config; -use codex_login::AuthManager; -use codex_login::default_client::build_reqwest_client; - -use super::PluginsManager; - -const GITHUB_API_BASE_URL: &str = "https://api.github.com"; -const GITHUB_API_ACCEPT_HEADER: &str = "application/vnd.github+json"; -const GITHUB_API_VERSION_HEADER: &str = "2022-11-28"; -const CURATED_PLUGINS_BACKUP_ARCHIVE_API_URL: &str = - "https://chatgpt.com/backend-api/plugins/export/curated"; -const OPENAI_PLUGINS_OWNER: &str = "openai"; -const OPENAI_PLUGINS_REPO: &str = "plugins"; -const CURATED_PLUGINS_RELATIVE_DIR: &str = ".tmp/plugins"; -const CURATED_PLUGINS_SHA_FILE: &str = ".tmp/plugins.sha"; -const CURATED_PLUGINS_BACKUP_ARCHIVE_FALLBACK_VERSION: &str = "export-backup"; -const CURATED_PLUGINS_GIT_TIMEOUT: Duration = Duration::from_secs(30); -const CURATED_PLUGINS_HTTP_TIMEOUT: Duration = Duration::from_secs(30); -const CURATED_PLUGINS_BACKUP_ARCHIVE_TIMEOUT: Duration = Duration::from_secs(30); -// Keep this comfortably above a normal sync attempt so we do not race another Codex process. -const CURATED_PLUGINS_STALE_TEMP_DIR_MAX_AGE: Duration = Duration::from_secs(10 * 60); const STARTUP_REMOTE_PLUGIN_SYNC_MARKER_FILE: &str = ".tmp/app-server-remote-plugin-sync-v1"; const STARTUP_REMOTE_PLUGIN_SYNC_PREREQUISITE_TIMEOUT: Duration = Duration::from_secs(10); -#[derive(Debug, Deserialize)] -struct GitHubRepositorySummary { - default_branch: String, -} - -#[derive(Debug, Deserialize)] -struct GitHubGitRefSummary { - object: GitHubGitRefObject, -} - -#[derive(Debug, Deserialize)] -struct GitHubGitRefObject { - sha: String, -} - -#[derive(Debug, Deserialize)] -struct CuratedPluginsBackupArchiveResponse { - download_url: String, -} - -pub(crate) fn curated_plugins_repo_path(codex_home: &Path) -> PathBuf { - codex_home.join(CURATED_PLUGINS_RELATIVE_DIR) -} - -pub(crate) fn read_curated_plugins_sha(codex_home: &Path) -> Option { - read_sha_file(curated_plugins_sha_path(codex_home).as_path()) -} - -fn curated_plugins_sha_path(codex_home: &Path) -> PathBuf { - codex_home.join(CURATED_PLUGINS_SHA_FILE) -} - -pub(crate) fn sync_openai_plugins_repo(codex_home: &Path) -> Result { - sync_openai_plugins_repo_with_transport_overrides( - codex_home, - "git", - GITHUB_API_BASE_URL, - CURATED_PLUGINS_BACKUP_ARCHIVE_API_URL, - ) -} - -fn sync_openai_plugins_repo_with_transport_overrides( - codex_home: &Path, - git_binary: &str, - api_base_url: &str, - backup_archive_api_url: &str, -) -> Result { - match sync_openai_plugins_repo_via_git(codex_home, git_binary) { - Ok(remote_sha) => { - emit_curated_plugins_startup_sync_metric("git", "success"); - emit_curated_plugins_startup_sync_final_metric("git", "success"); - Ok(remote_sha) - } - Err(err) => { - emit_curated_plugins_startup_sync_metric("git", "failure"); - warn!( - error = %err, - git_binary, - "git sync failed for curated plugin sync; falling back to GitHub HTTP" - ); - match sync_openai_plugins_repo_via_http(codex_home, api_base_url) { - Ok(remote_sha) => { - emit_curated_plugins_startup_sync_metric("http", "success"); - emit_curated_plugins_startup_sync_final_metric("http", "success"); - Ok(remote_sha) - } - Err(http_err) => { - emit_curated_plugins_startup_sync_metric("http", "failure"); - if has_local_curated_plugins_snapshot(codex_home) { - emit_curated_plugins_startup_sync_final_metric("http", "failure"); - warn!( - error = %http_err, - "GitHub HTTP sync failed for curated plugin sync; skipping export archive fallback because a local curated plugins snapshot already exists" - ); - Err(format!( - "git sync failed for curated plugin sync: {err}; GitHub HTTP sync failed for curated plugin sync: {http_err}; export archive fallback skipped because a local curated plugins snapshot already exists" - )) - } else { - // The export archive is a lagging backup path. Only use it to bootstrap a - // missing local curated snapshot, never to refresh an existing one. - warn!( - error = %http_err, - backup_archive_api_url, - "GitHub HTTP sync failed for curated plugin sync; falling back to export archive" - ); - let result = sync_openai_plugins_repo_via_backup_archive( - codex_home, - backup_archive_api_url, - ); - let status = if result.is_ok() { "success" } else { "failure" }; - emit_curated_plugins_startup_sync_metric("export_archive", status); - emit_curated_plugins_startup_sync_final_metric("export_archive", status); - result.map_err(|export_err| { - format!( - "git sync failed for curated plugin sync: {err}; GitHub HTTP sync failed for curated plugin sync: {http_err}; export archive sync failed for curated plugin sync: {export_err}" - ) - }) - } - } - } - } - } -} - -fn sync_openai_plugins_repo_via_git(codex_home: &Path, git_binary: &str) -> Result { - let repo_path = curated_plugins_repo_path(codex_home); - let sha_path = codex_home.join(CURATED_PLUGINS_SHA_FILE); - let remote_sha = git_ls_remote_head_sha(git_binary)?; - let local_sha = read_local_git_or_sha_file(&repo_path, &sha_path, git_binary); - - if local_sha.as_deref() == Some(remote_sha.as_str()) && repo_path.join(".git").is_dir() { - return Ok(remote_sha); - } - - let staged_repo_dir = prepare_curated_repo_parent_and_temp_dir(&repo_path)?; - let clone_output = run_git_command_with_timeout( - Command::new(git_binary) - .env("GIT_OPTIONAL_LOCKS", "0") - .arg("clone") - .arg("--depth") - .arg("1") - .arg("https://github.com/openai/plugins.git") - .arg(staged_repo_dir.path()), - "git clone curated plugins repo", - CURATED_PLUGINS_GIT_TIMEOUT, - )?; - ensure_git_success(&clone_output, "git clone curated plugins repo")?; - - let cloned_sha = git_head_sha(staged_repo_dir.path(), git_binary)?; - if cloned_sha != remote_sha { - return Err(format!( - "curated plugins clone HEAD mismatch: expected {remote_sha}, got {cloned_sha}" - )); - } - - ensure_marketplace_manifest_exists(staged_repo_dir.path())?; - activate_curated_repo(&repo_path, staged_repo_dir)?; - write_curated_plugins_sha(&sha_path, &remote_sha)?; - Ok(remote_sha) -} - -fn sync_openai_plugins_repo_via_http( - codex_home: &Path, - api_base_url: &str, -) -> Result { - let repo_path = curated_plugins_repo_path(codex_home); - let sha_path = codex_home.join(CURATED_PLUGINS_SHA_FILE); - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|err| format!("failed to create curated plugins sync runtime: {err}"))?; - let remote_sha = runtime.block_on(fetch_curated_repo_remote_sha(api_base_url))?; - let local_sha = read_sha_file(&sha_path); - - if local_sha.as_deref() == Some(remote_sha.as_str()) && repo_path.is_dir() { - return Ok(remote_sha); - } - - let staged_repo_dir = prepare_curated_repo_parent_and_temp_dir(&repo_path)?; - let zipball_bytes = runtime.block_on(fetch_curated_repo_zipball(api_base_url, &remote_sha))?; - extract_zipball_to_dir(&zipball_bytes, staged_repo_dir.path())?; - ensure_marketplace_manifest_exists(staged_repo_dir.path())?; - activate_curated_repo(&repo_path, staged_repo_dir)?; - write_curated_plugins_sha(&sha_path, &remote_sha)?; - Ok(remote_sha) -} - -fn sync_openai_plugins_repo_via_backup_archive( - codex_home: &Path, - backup_archive_api_url: &str, -) -> Result { - let repo_path = curated_plugins_repo_path(codex_home); - let sha_path = curated_plugins_sha_path(codex_home); - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|err| format!("failed to create curated plugins sync runtime: {err}"))?; - let staged_repo_dir = prepare_curated_repo_parent_and_temp_dir(&repo_path)?; - let zipball_bytes = runtime.block_on(fetch_curated_repo_backup_archive_zip( - backup_archive_api_url, - ))?; - extract_zipball_to_dir(&zipball_bytes, staged_repo_dir.path())?; - ensure_marketplace_manifest_exists(staged_repo_dir.path())?; - let export_version = read_extracted_backup_archive_git_sha(staged_repo_dir.path())? - .unwrap_or_else(|| CURATED_PLUGINS_BACKUP_ARCHIVE_FALLBACK_VERSION.to_string()); - activate_curated_repo(&repo_path, staged_repo_dir)?; - write_curated_plugins_sha(&sha_path, &export_version)?; - Ok(export_version) -} - -pub(super) fn start_startup_remote_plugin_sync_once( +pub(crate) fn start_startup_remote_plugin_sync_once( manager: Arc, codex_home: PathBuf, config: Config, @@ -290,13 +74,6 @@ fn startup_remote_plugin_sync_marker_path(codex_home: &Path) -> PathBuf { codex_home.join(STARTUP_REMOTE_PLUGIN_SYNC_MARKER_FILE) } -fn has_local_curated_plugins_snapshot(codex_home: &Path) -> bool { - curated_plugins_repo_path(codex_home) - .join(".agents/plugins/marketplace.json") - .is_file() - && codex_home.join(CURATED_PLUGINS_SHA_FILE).is_file() -} - async fn wait_for_startup_remote_plugin_sync_prerequisites(codex_home: &Path) -> bool { let deadline = tokio::time::Instant::now() + STARTUP_REMOTE_PLUGIN_SYNC_PREREQUISITE_TIMEOUT; loop { @@ -318,711 +95,6 @@ async fn write_startup_remote_plugin_sync_marker(codex_home: &Path) -> std::io:: tokio::fs::write(marker_path, b"ok\n").await } -fn prepare_curated_repo_parent_and_temp_dir(repo_path: &Path) -> Result { - let Some(parent) = repo_path.parent() else { - return Err(format!( - "failed to determine curated plugins parent directory for {}", - repo_path.display() - )); - }; - std::fs::create_dir_all(parent).map_err(|err| { - format!( - "failed to create curated plugins parent directory {}: {err}", - parent.display() - ) - })?; - remove_stale_curated_repo_temp_dirs(parent, CURATED_PLUGINS_STALE_TEMP_DIR_MAX_AGE); - - let clone_dir = tempfile::Builder::new() - .prefix("plugins-clone-") - .tempdir_in(parent) - .map_err(|err| { - format!( - "failed to create temporary curated plugins directory in {}: {err}", - parent.display() - ) - })?; - Ok(clone_dir) -} - -fn remove_stale_curated_repo_temp_dirs(parent: &Path, max_age: Duration) { - let entries = match std::fs::read_dir(parent) { - Ok(entries) => entries, - Err(err) => { - warn!( - error = %err, - parent = %parent.display(), - "failed to list curated plugins temp directory parent for stale cleanup" - ); - return; - } - }; - - for entry in entries.flatten() { - let file_type = match entry.file_type() { - Ok(file_type) => file_type, - Err(err) => { - warn!( - error = %err, - path = %entry.path().display(), - "failed to inspect curated plugins temp directory entry" - ); - continue; - } - }; - if !file_type.is_dir() { - continue; - } - - let path = entry.path(); - let is_plugins_clone_dir = path - .file_name() - .and_then(|name| name.to_str()) - .is_some_and(|name| name.starts_with("plugins-clone-")); - if !is_plugins_clone_dir { - continue; - } - - let metadata = match entry.metadata() { - Ok(metadata) => metadata, - Err(err) => { - warn!( - error = %err, - path = %path.display(), - "failed to read curated plugins temp directory metadata" - ); - continue; - } - }; - let modified = match metadata.modified() { - Ok(modified) => modified, - Err(err) => { - warn!( - error = %err, - path = %path.display(), - "failed to read curated plugins temp directory modification time" - ); - continue; - } - }; - let age = match modified.elapsed() { - Ok(age) => age, - Err(err) => { - warn!( - error = %err, - path = %path.display(), - "failed to compute curated plugins temp directory age" - ); - continue; - } - }; - if age < max_age { - continue; - } - - if let Err(err) = std::fs::remove_dir_all(&path) { - warn!( - error = %err, - path = %path.display(), - "failed to remove stale curated plugins temp directory" - ); - } - } -} - -fn emit_curated_plugins_startup_sync_metric(transport: &'static str, status: &'static str) { - emit_curated_plugins_startup_sync_counter( - CURATED_PLUGINS_STARTUP_SYNC_METRIC, - transport, - status, - ); -} - -fn emit_curated_plugins_startup_sync_final_metric(transport: &'static str, status: &'static str) { - emit_curated_plugins_startup_sync_counter( - CURATED_PLUGINS_STARTUP_SYNC_FINAL_METRIC, - transport, - status, - ); -} - -fn emit_curated_plugins_startup_sync_counter( - metric_name: &str, - transport: &'static str, - status: &'static str, -) { - let Some(metrics) = codex_otel::global() else { - return; - }; - let tags = [("transport", transport), ("status", status)]; - let _ = metrics.counter(metric_name, /*inc*/ 1, &tags); -} - -fn ensure_marketplace_manifest_exists(repo_path: &Path) -> Result<(), String> { - if repo_path.join(".agents/plugins/marketplace.json").is_file() { - return Ok(()); - } - Err(format!( - "curated plugins archive missing marketplace manifest at {}", - repo_path.join(".agents/plugins/marketplace.json").display() - )) -} - -fn activate_curated_repo(repo_path: &Path, staged_repo_dir: TempDir) -> Result<(), String> { - let staged_repo_path = staged_repo_dir.path(); - if repo_path.exists() { - let parent = repo_path.parent().ok_or_else(|| { - format!( - "failed to determine curated plugins parent directory for {}", - repo_path.display() - ) - })?; - let backup_dir = tempfile::Builder::new() - .prefix("plugins-backup-") - .tempdir_in(parent) - .map_err(|err| { - format!( - "failed to create curated plugins backup directory in {}: {err}", - parent.display() - ) - })?; - let backup_repo_path = backup_dir.path().join("repo"); - - std::fs::rename(repo_path, &backup_repo_path).map_err(|err| { - format!( - "failed to move previous curated plugins repo out of the way at {}: {err}", - repo_path.display() - ) - })?; - - if let Err(err) = std::fs::rename(staged_repo_path, repo_path) { - let rollback_result = std::fs::rename(&backup_repo_path, repo_path); - return match rollback_result { - Ok(()) => Err(format!( - "failed to activate new curated plugins repo at {}: {err}", - repo_path.display() - )), - Err(rollback_err) => { - let backup_path = backup_dir.keep().join("repo"); - Err(format!( - "failed to activate new curated plugins repo at {}: {err}; failed to restore previous repo (left at {}): {rollback_err}", - repo_path.display(), - backup_path.display() - )) - } - }; - } - } else { - std::fs::rename(staged_repo_path, repo_path).map_err(|err| { - format!( - "failed to activate curated plugins repo at {}: {err}", - repo_path.display() - ) - })?; - } - - Ok(()) -} - -fn write_curated_plugins_sha(sha_path: &Path, remote_sha: &str) -> Result<(), String> { - if let Some(parent) = sha_path.parent() { - std::fs::create_dir_all(parent).map_err(|err| { - format!( - "failed to create curated plugins sha directory {}: {err}", - parent.display() - ) - })?; - } - std::fs::write(sha_path, format!("{remote_sha}\n")).map_err(|err| { - format!( - "failed to write curated plugins sha file {}: {err}", - sha_path.display() - ) - }) -} - -fn read_local_git_or_sha_file( - repo_path: &Path, - sha_path: &Path, - git_binary: &str, -) -> Option { - if repo_path.join(".git").is_dir() - && let Ok(sha) = git_head_sha(repo_path, git_binary) - { - return Some(sha); - } - - read_sha_file(sha_path) -} - -fn git_ls_remote_head_sha(git_binary: &str) -> Result { - let output = run_git_command_with_timeout( - Command::new(git_binary) - .env("GIT_OPTIONAL_LOCKS", "0") - .arg("ls-remote") - .arg("https://github.com/openai/plugins.git") - .arg("HEAD"), - "git ls-remote curated plugins repo", - CURATED_PLUGINS_GIT_TIMEOUT, - )?; - ensure_git_success(&output, "git ls-remote curated plugins repo")?; - - let stdout = String::from_utf8_lossy(&output.stdout); - let Some(first_line) = stdout.lines().next() else { - return Err("git ls-remote returned empty output for curated plugins repo".to_string()); - }; - let Some((sha, _)) = first_line.split_once('\t') else { - return Err(format!( - "unexpected git ls-remote output for curated plugins repo: {first_line}" - )); - }; - if sha.is_empty() { - return Err("git ls-remote returned empty sha for curated plugins repo".to_string()); - } - Ok(sha.to_string()) -} - -fn git_head_sha(repo_path: &Path, git_binary: &str) -> Result { - let output = Command::new(git_binary) - .env("GIT_OPTIONAL_LOCKS", "0") - .arg("-C") - .arg(repo_path) - .arg("rev-parse") - .arg("HEAD") - .output() - .map_err(|err| { - format!( - "failed to run git rev-parse HEAD in {}: {err}", - repo_path.display() - ) - })?; - ensure_git_success(&output, "git rev-parse HEAD")?; - - let sha = String::from_utf8_lossy(&output.stdout).trim().to_string(); - if sha.is_empty() { - return Err(format!( - "git rev-parse HEAD returned empty output in {}", - repo_path.display() - )); - } - Ok(sha) -} - -fn run_git_command_with_timeout( - command: &mut Command, - context: &str, - timeout: Duration, -) -> Result { - let mut child = command - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .map_err(|err| format!("failed to run {context}: {err}"))?; - - let start = std::time::Instant::now(); - loop { - match child.try_wait() { - Ok(Some(_)) => { - return child - .wait_with_output() - .map_err(|err| format!("failed to wait for {context}: {err}")); - } - Ok(None) => {} - Err(err) => return Err(format!("failed to poll {context}: {err}")), - } - - if start.elapsed() >= timeout { - match child.try_wait() { - Ok(Some(_)) => { - return child - .wait_with_output() - .map_err(|err| format!("failed to wait for {context}: {err}")); - } - Ok(None) => {} - Err(err) => return Err(format!("failed to poll {context}: {err}")), - } - - let _ = child.kill(); - let output = child - .wait_with_output() - .map_err(|err| format!("failed to wait for {context} after timeout: {err}"))?; - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); - return if stderr.is_empty() { - Err(format!("{context} timed out after {}s", timeout.as_secs())) - } else { - Err(format!( - "{context} timed out after {}s: {stderr}", - timeout.as_secs() - )) - }; - } - - std::thread::sleep(Duration::from_millis(100)); - } -} - -fn ensure_git_success(output: &Output, context: &str) -> Result<(), String> { - if output.status.success() { - return Ok(()); - } - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); - if stderr.is_empty() { - Err(format!("{context} failed with status {}", output.status)) - } else { - Err(format!( - "{context} failed with status {}: {stderr}", - output.status - )) - } -} - -async fn fetch_curated_repo_remote_sha(api_base_url: &str) -> Result { - let api_base_url = api_base_url.trim_end_matches('/'); - let repo_url = format!("{api_base_url}/repos/{OPENAI_PLUGINS_OWNER}/{OPENAI_PLUGINS_REPO}"); - let client = build_reqwest_client(); - let repo_body = fetch_github_text(&client, &repo_url, "get curated plugins repository").await?; - let repo_summary: GitHubRepositorySummary = - serde_json::from_str(&repo_body).map_err(|err| { - format!("failed to parse curated plugins repository response from {repo_url}: {err}") - })?; - if repo_summary.default_branch.is_empty() { - return Err(format!( - "curated plugins repository response from {repo_url} did not include a default branch" - )); - } - - let git_ref_url = format!("{repo_url}/git/ref/heads/{}", repo_summary.default_branch); - let git_ref_body = - fetch_github_text(&client, &git_ref_url, "get curated plugins HEAD ref").await?; - let git_ref: GitHubGitRefSummary = serde_json::from_str(&git_ref_body).map_err(|err| { - format!("failed to parse curated plugins ref response from {git_ref_url}: {err}") - })?; - if git_ref.object.sha.is_empty() { - return Err(format!( - "curated plugins ref response from {git_ref_url} did not include a HEAD sha" - )); - } - - Ok(git_ref.object.sha) -} - -async fn fetch_curated_repo_zipball( - api_base_url: &str, - remote_sha: &str, -) -> Result, String> { - let api_base_url = api_base_url.trim_end_matches('/'); - let repo_url = format!("{api_base_url}/repos/{OPENAI_PLUGINS_OWNER}/{OPENAI_PLUGINS_REPO}"); - let zipball_url = format!("{repo_url}/zipball/{remote_sha}"); - let client = build_reqwest_client(); - fetch_github_bytes(&client, &zipball_url, "download curated plugins archive").await -} - -async fn fetch_curated_repo_backup_archive_zip( - backup_archive_api_url: &str, -) -> Result, String> { - let client = build_reqwest_client(); - let export_body = fetch_public_text( - &client, - backup_archive_api_url, - "get curated plugins export archive metadata", - ) - .await?; - let export_response: CuratedPluginsBackupArchiveResponse = serde_json::from_str(&export_body) - .map_err(|err| { - format!( - "failed to parse curated plugins backup archive response from {backup_archive_api_url}: {err}" - ) - })?; - if export_response.download_url.is_empty() { - return Err(format!( - "curated plugins backup archive response from {backup_archive_api_url} did not include a download URL" - )); - } - - fetch_public_bytes( - &client, - &export_response.download_url, - "download curated plugins export archive", - ) - .await -} - -fn read_extracted_backup_archive_git_sha(repo_path: &Path) -> Result, String> { - let git_dir = repo_path.join(".git"); - if !git_dir.is_dir() { - return Ok(None); - } - - let head_path = git_dir.join("HEAD"); - let head = std::fs::read_to_string(&head_path).map_err(|err| { - format!( - "failed to read curated plugins backup archive git HEAD {}: {err}", - head_path.display() - ) - })?; - let head = head.trim(); - if head.is_empty() { - return Err(format!( - "curated plugins backup archive git HEAD is empty at {}", - head_path.display() - )); - } - - if let Some(reference) = head.strip_prefix("ref: ") { - let reference = validate_backup_archive_git_ref(reference.trim())?; - return read_git_ref_sha(&git_dir, reference).map(Some); - } - - Ok(Some(head.to_string())) -} - -fn validate_backup_archive_git_ref(reference: &str) -> Result<&str, String> { - if !reference.starts_with("refs/") { - return Err(format!( - "curated plugins backup archive git ref must stay under refs/: {reference}" - )); - } - - let path = Path::new(reference); - if path.is_absolute() { - return Err(format!( - "curated plugins backup archive git ref must be relative: {reference}" - )); - } - - for component in path.components() { - match component { - std::path::Component::Normal(_) => {} - _ => { - return Err(format!( - "curated plugins backup archive git ref contains invalid path components: {reference}" - )); - } - } - } - - Ok(reference) -} - -fn read_git_ref_sha(git_dir: &Path, reference: &str) -> Result { - let ref_path = git_dir.join(reference); - if let Ok(sha) = std::fs::read_to_string(&ref_path) { - let sha = sha.trim(); - if sha.is_empty() { - return Err(format!( - "curated plugins backup archive git ref {reference} is empty at {}", - ref_path.display() - )); - } - return Ok(sha.to_string()); - } - - let packed_refs_path = git_dir.join("packed-refs"); - if let Ok(packed_refs) = std::fs::read_to_string(&packed_refs_path) - && let Some(sha) = packed_refs.lines().find_map(|line| { - let trimmed = line.trim(); - if trimmed.is_empty() || trimmed.starts_with('#') || trimmed.starts_with('^') { - return None; - } - let (sha, candidate_ref) = trimmed.split_once(' ')?; - (candidate_ref == reference).then_some(sha.to_string()) - }) - { - return Ok(sha); - } - - Err(format!( - "failed to resolve curated plugins backup archive git ref {reference} from {}", - git_dir.display() - )) -} - -async fn fetch_github_text(client: &Client, url: &str, context: &str) -> Result { - let response = github_request(client, url) - .send() - .await - .map_err(|err| format!("failed to {context} from {url}: {err}"))?; - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - if !status.is_success() { - return Err(format!( - "{context} from {url} failed with status {status}: {body}" - )); - } - Ok(body) -} - -async fn fetch_github_bytes(client: &Client, url: &str, context: &str) -> Result, String> { - let response = github_request(client, url) - .send() - .await - .map_err(|err| format!("failed to {context} from {url}: {err}"))?; - let status = response.status(); - let body = response - .bytes() - .await - .map_err(|err| format!("failed to read {context} response from {url}: {err}"))?; - if !status.is_success() { - let body_text = String::from_utf8_lossy(&body); - return Err(format!( - "{context} from {url} failed with status {status}: {body_text}" - )); - } - Ok(body.to_vec()) -} - -async fn fetch_public_text(client: &Client, url: &str, context: &str) -> Result { - let response = client - .get(url) - .timeout(CURATED_PLUGINS_BACKUP_ARCHIVE_TIMEOUT) - .send() - .await - .map_err(|err| format!("failed to {context} from {url}: {err}"))?; - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - if !status.is_success() { - return Err(format!( - "{context} from {url} failed with status {status}: {body}" - )); - } - Ok(body) -} - -async fn fetch_public_bytes(client: &Client, url: &str, context: &str) -> Result, String> { - let response = client - .get(url) - .timeout(CURATED_PLUGINS_BACKUP_ARCHIVE_TIMEOUT) - .send() - .await - .map_err(|err| format!("failed to {context} from {url}: {err}"))?; - let status = response.status(); - let body = response - .bytes() - .await - .map_err(|err| format!("failed to read {context} response from {url}: {err}"))?; - if !status.is_success() { - let body_text = String::from_utf8_lossy(&body); - return Err(format!( - "{context} from {url} failed with status {status}: {body_text}" - )); - } - Ok(body.to_vec()) -} - -fn github_request(client: &Client, url: &str) -> reqwest::RequestBuilder { - client - .get(url) - .timeout(CURATED_PLUGINS_HTTP_TIMEOUT) - .header("accept", GITHUB_API_ACCEPT_HEADER) - .header("x-github-api-version", GITHUB_API_VERSION_HEADER) -} - -fn read_sha_file(sha_path: &Path) -> Option { - std::fs::read_to_string(sha_path) - .ok() - .map(|sha| sha.trim().to_string()) - .filter(|sha| !sha.is_empty()) -} - -fn extract_zipball_to_dir(bytes: &[u8], destination: &Path) -> Result<(), String> { - std::fs::create_dir_all(destination).map_err(|err| { - format!( - "failed to create curated plugins extraction directory {}: {err}", - destination.display() - ) - })?; - - let cursor = std::io::Cursor::new(bytes); - let mut archive = ZipArchive::new(cursor) - .map_err(|err| format!("failed to open curated plugins zip archive: {err}"))?; - - for index in 0..archive.len() { - let mut entry = archive - .by_index(index) - .map_err(|err| format!("failed to read curated plugins zip entry: {err}"))?; - let Some(relative_path) = entry.enclosed_name() else { - return Err(format!( - "curated plugins zip entry `{}` escapes extraction root", - entry.name() - )); - }; - - let mut components = relative_path.components(); - let Some(std::path::Component::Normal(_)) = components.next() else { - continue; - }; - - let output_relative = components.fold(PathBuf::new(), |mut path, component| { - if let std::path::Component::Normal(segment) = component { - path.push(segment); - } - path - }); - if output_relative.as_os_str().is_empty() { - continue; - } - - let output_path = destination.join(&output_relative); - if entry.is_dir() { - std::fs::create_dir_all(&output_path).map_err(|err| { - format!( - "failed to create curated plugins directory {}: {err}", - output_path.display() - ) - })?; - continue; - } - - if let Some(parent) = output_path.parent() { - std::fs::create_dir_all(parent).map_err(|err| { - format!( - "failed to create curated plugins directory {}: {err}", - parent.display() - ) - })?; - } - let mut output = std::fs::File::create(&output_path).map_err(|err| { - format!( - "failed to create curated plugins file {}: {err}", - output_path.display() - ) - })?; - std::io::copy(&mut entry, &mut output).map_err(|err| { - format!( - "failed to write curated plugins file {}: {err}", - output_path.display() - ) - })?; - apply_zip_permissions(&entry, &output_path)?; - } - - Ok(()) -} - -#[cfg(unix)] -fn apply_zip_permissions(entry: &zip::read::ZipFile<'_>, output_path: &Path) -> Result<(), String> { - use std::os::unix::fs::PermissionsExt; - - let Some(mode) = entry.unix_mode() else { - return Ok(()); - }; - std::fs::set_permissions(output_path, std::fs::Permissions::from_mode(mode)).map_err(|err| { - format!( - "failed to set permissions on curated plugins file {}: {err}", - output_path.display() - ) - }) -} - -#[cfg(not(unix))] -fn apply_zip_permissions( - _entry: &zip::read::ZipFile<'_>, - _output_path: &Path, -) -> Result<(), String> { - Ok(()) -} - #[cfg(test)] #[path = "startup_sync_tests.rs"] mod tests; diff --git a/codex-rs/core/src/plugins/startup_sync_tests.rs b/codex-rs/core/src/plugins/startup_sync_tests.rs index 74dd0d0fdb..8dc2f748f6 100644 --- a/codex-rs/core/src/plugins/startup_sync_tests.rs +++ b/codex-rs/core/src/plugins/startup_sync_tests.rs @@ -1,14 +1,16 @@ use super::*; use crate::config::CONFIG_TOML_FILE; +use crate::plugins::PluginsManager; use crate::plugins::test_support::TEST_CURATED_PLUGIN_SHA; use crate::plugins::test_support::write_curated_plugin_sha; use crate::plugins::test_support::write_file; use crate::plugins::test_support::write_openai_curated_marketplace; +use codex_core_plugins::startup_sync::curated_plugins_repo_path; +use codex_login::AuthManager; use codex_login::CodexAuth; use pretty_assertions::assert_eq; -use std::io::Write; -use std::path::Path; -use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; use tempfile::tempdir; use wiremock::Mock; use wiremock::MockServer; @@ -16,627 +18,6 @@ use wiremock::ResponseTemplate; use wiremock::matchers::header; use wiremock::matchers::method; use wiremock::matchers::path; -use zip::ZipWriter; -use zip::write::SimpleFileOptions; - -fn has_plugins_clone_dirs(codex_home: &Path) -> bool { - let Ok(entries) = std::fs::read_dir(codex_home.join(".tmp")) else { - return false; - }; - - entries.flatten().any(|entry| { - let path = entry.path(); - path.is_dir() - && path - .file_name() - .and_then(|name| name.to_str()) - .is_some_and(|name| name.starts_with("plugins-clone-")) - }) -} - -#[cfg(unix)] -fn write_executable_script(path: &Path, contents: &str) { - #[cfg(unix)] - use std::os::unix::fs::PermissionsExt; - - std::fs::write(path, contents).expect("write script"); - #[cfg(unix)] - { - let mut permissions = std::fs::metadata(path).expect("metadata").permissions(); - permissions.set_mode(0o755); - std::fs::set_permissions(path, permissions).expect("chmod"); - } -} - -async fn mount_github_repo_and_ref(server: &MockServer, sha: &str) { - Mock::given(method("GET")) - .and(path("/repos/openai/plugins")) - .respond_with(ResponseTemplate::new(200).set_body_string(r#"{"default_branch":"main"}"#)) - .mount(server) - .await; - Mock::given(method("GET")) - .and(path("/repos/openai/plugins/git/ref/heads/main")) - .respond_with( - ResponseTemplate::new(200) - .set_body_string(format!(r#"{{"object":{{"sha":"{sha}"}}}}"#)), - ) - .mount(server) - .await; -} - -async fn mount_github_zipball(server: &MockServer, sha: &str, bytes: Vec) { - Mock::given(method("GET")) - .and(path(format!("/repos/openai/plugins/zipball/{sha}"))) - .respond_with( - ResponseTemplate::new(200) - .insert_header("content-type", "application/zip") - .set_body_bytes(bytes), - ) - .mount(server) - .await; -} - -async fn mount_export_archive(server: &MockServer, bytes: Vec) -> String { - let export_api_url = format!("{}/backend-api/plugins/export/curated", server.uri()); - Mock::given(method("GET")) - .and(path("/backend-api/plugins/export/curated")) - .respond_with(ResponseTemplate::new(200).set_body_string(format!( - r#"{{"download_url":"{}/files/curated-plugins.zip"}}"#, - server.uri() - ))) - .mount(server) - .await; - Mock::given(method("GET")) - .and(path("/files/curated-plugins.zip")) - .respond_with( - ResponseTemplate::new(200) - .insert_header("content-type", "application/zip") - .set_body_bytes(bytes), - ) - .mount(server) - .await; - export_api_url -} - -async fn run_sync_with_transport_overrides( - codex_home: PathBuf, - git_binary: impl Into, - api_base_url: impl Into, - backup_archive_api_url: impl Into, -) -> Result { - let git_binary = git_binary.into(); - let api_base_url = api_base_url.into(); - let backup_archive_api_url = backup_archive_api_url.into(); - tokio::task::spawn_blocking(move || { - sync_openai_plugins_repo_with_transport_overrides( - codex_home.as_path(), - &git_binary, - &api_base_url, - &backup_archive_api_url, - ) - }) - .await - .expect("sync task should join") -} - -async fn run_http_sync( - codex_home: PathBuf, - api_base_url: impl Into, -) -> Result { - let api_base_url = api_base_url.into(); - tokio::task::spawn_blocking(move || { - sync_openai_plugins_repo_via_http(codex_home.as_path(), &api_base_url) - }) - .await - .expect("sync task should join") -} - -fn assert_curated_gmail_repo(repo_path: &Path) { - assert!(repo_path.join(".agents/plugins/marketplace.json").is_file()); - assert!( - repo_path - .join("plugins/gmail/.codex-plugin/plugin.json") - .is_file() - ); -} - -#[test] -fn curated_plugins_repo_path_uses_codex_home_tmp_dir() { - let tmp = tempdir().expect("tempdir"); - assert_eq!( - curated_plugins_repo_path(tmp.path()), - tmp.path().join(".tmp/plugins") - ); -} - -#[test] -fn read_curated_plugins_sha_reads_trimmed_sha_file() { - let tmp = tempdir().expect("tempdir"); - std::fs::create_dir_all(tmp.path().join(".tmp")).expect("create tmp"); - std::fs::write(tmp.path().join(".tmp/plugins.sha"), "abc123\n").expect("write sha"); - - assert_eq!( - read_curated_plugins_sha(tmp.path()).as_deref(), - Some("abc123") - ); -} - -#[cfg(unix)] -#[test] -fn remove_stale_curated_repo_temp_dirs_removes_only_matching_directories() { - use std::os::unix::ffi::OsStrExt; - use std::time::SystemTime; - - fn set_dir_mtime(path: &Path, age: Duration) -> Result<(), Box> { - let now = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH)?; - let modified_at = now.saturating_sub(age); - let tv_sec = i64::try_from(modified_at.as_secs())?; - let ts = libc::timespec { tv_sec, tv_nsec: 0 }; - let times = [ts, ts]; - let c_path = std::ffi::CString::new(path.as_os_str().as_bytes())?; - let result = unsafe { libc::utimensat(libc::AT_FDCWD, c_path.as_ptr(), times.as_ptr(), 0) }; - if result != 0 { - return Err(std::io::Error::last_os_error().into()); - } - Ok(()) - } - - let tmp = tempdir().expect("tempdir"); - let parent = tmp.path().join(".tmp"); - let stale_clone_dir = parent.join("plugins-clone-stale"); - let fresh_clone_dir = parent.join("plugins-clone-fresh"); - let unrelated_dir = parent.join("plugins-cache"); - - std::fs::create_dir_all(&stale_clone_dir).expect("create stale clone dir"); - std::fs::create_dir_all(&fresh_clone_dir).expect("create fresh clone dir"); - std::fs::create_dir_all(&unrelated_dir).expect("create unrelated dir"); - set_dir_mtime( - &stale_clone_dir, - CURATED_PLUGINS_STALE_TEMP_DIR_MAX_AGE + Duration::from_secs(60), - ) - .expect("age stale clone dir"); - set_dir_mtime(&fresh_clone_dir, Duration::ZERO).expect("age fresh clone dir"); - - remove_stale_curated_repo_temp_dirs(&parent, CURATED_PLUGINS_STALE_TEMP_DIR_MAX_AGE); - - assert!(!stale_clone_dir.exists()); - assert!(fresh_clone_dir.is_dir()); - assert!(unrelated_dir.is_dir()); -} - -#[cfg(unix)] -#[test] -fn sync_openai_plugins_repo_prefers_git_when_available() { - let tmp = tempdir().expect("tempdir"); - let bin_dir = tempfile::Builder::new() - .prefix("fake-git-") - .tempdir() - .expect("tempdir"); - let git_path = bin_dir.path().join("git"); - let sha = "0123456789abcdef0123456789abcdef01234567"; - - write_executable_script( - &git_path, - &format!( - r#"#!/bin/sh -if [ "$1" = "ls-remote" ]; then - printf '%s\tHEAD\n' "{sha}" - exit 0 -fi -if [ "$1" = "clone" ]; then - dest="$5" - mkdir -p "$dest/.git" "$dest/.agents/plugins" "$dest/plugins/gmail/.codex-plugin" - cat > "$dest/.agents/plugins/marketplace.json" <<'EOF' -{{"name":"openai-curated","plugins":[{{"name":"gmail","source":{{"source":"local","path":"./plugins/gmail"}}}}]}} -EOF - printf '%s\n' '{{"name":"gmail"}}' > "$dest/plugins/gmail/.codex-plugin/plugin.json" - exit 0 -fi -if [ "$1" = "-C" ] && [ "$3" = "rev-parse" ] && [ "$4" = "HEAD" ]; then - printf '%s\n' "{sha}" - exit 0 -fi -echo "unexpected git invocation: $@" >&2 -exit 1 -"# - ), - ); - - let synced_sha = sync_openai_plugins_repo_with_transport_overrides( - tmp.path(), - git_path.to_str().expect("utf8 path"), - "http://127.0.0.1:9", - "http://127.0.0.1:9/backend-api/plugins/export/curated", - ) - .expect("git sync should succeed"); - - assert_eq!(synced_sha, sha); - let repo_path = curated_plugins_repo_path(tmp.path()); - assert!(repo_path.join(".git").is_dir()); - assert_curated_gmail_repo(&repo_path); - assert_eq!(read_curated_plugins_sha(tmp.path()).as_deref(), Some(sha)); -} - -#[cfg(unix)] -#[test] -fn sync_openai_plugins_repo_via_git_succeeds_with_local_rewritten_remote() { - let tmp = tempdir().expect("tempdir"); - let repo_root = tempfile::Builder::new() - .prefix("curated-repo-success-") - .tempdir() - .expect("tempdir"); - let work_repo = repo_root.path().join("work/plugins"); - let remote_repo = repo_root.path().join("remotes/openai/plugins.git"); - std::fs::create_dir_all(work_repo.join(".agents/plugins")).expect("create marketplace dir"); - std::fs::create_dir_all(work_repo.join("plugins/gmail/.codex-plugin")) - .expect("create plugin dir"); - std::fs::write( - work_repo.join(".agents/plugins/marketplace.json"), - r#"{"name":"openai-curated","plugins":[{"name":"gmail","source":{"source":"local","path":"./plugins/gmail"}}]}"#, - ) - .expect("write marketplace"); - std::fs::write( - work_repo.join("plugins/gmail/.codex-plugin/plugin.json"), - r#"{"name":"gmail"}"#, - ) - .expect("write plugin manifest"); - - let init_status = Command::new("git") - .arg("-C") - .arg(&work_repo) - .arg("init") - .status() - .expect("run git init"); - assert!(init_status.success()); - - let add_status = Command::new("git") - .arg("-C") - .arg(&work_repo) - .arg("add") - .arg(".") - .status() - .expect("run git add"); - assert!(add_status.success()); - - let commit_status = Command::new("git") - .arg("-C") - .arg(&work_repo) - .arg("-c") - .arg("user.name=Codex Test") - .arg("-c") - .arg("user.email=codex@example.com") - .arg("commit") - .arg("-m") - .arg("init") - .status() - .expect("run git commit"); - assert!(commit_status.success()); - - std::fs::create_dir_all(remote_repo.parent().expect("remote parent")) - .expect("create remote parent"); - let clone_status = Command::new("git") - .arg("clone") - .arg("--bare") - .arg(&work_repo) - .arg(&remote_repo) - .status() - .expect("run git clone --bare"); - assert!(clone_status.success()); - - let sha_output = Command::new("git") - .arg("-C") - .arg(&work_repo) - .arg("rev-parse") - .arg("HEAD") - .output() - .expect("run git rev-parse"); - assert!(sha_output.status.success()); - let sha = String::from_utf8_lossy(&sha_output.stdout) - .trim() - .to_string(); - - let git_config_path = repo_root.path().join("git-rewrite.conf"); - std::fs::write( - &git_config_path, - format!( - "[url \"file://{}/\"]\n insteadOf = https://github.com/\n", - repo_root.path().join("remotes").display() - ), - ) - .expect("write git config"); - - let bin_dir = tempfile::Builder::new() - .prefix("git-rewrite-wrapper-") - .tempdir() - .expect("tempdir"); - let git_wrapper = bin_dir.path().join("git"); - write_executable_script( - &git_wrapper, - &format!( - "#!/bin/sh\nGIT_CONFIG_GLOBAL='{}' exec git \"$@\"\n", - git_config_path.display() - ), - ); - - let synced_sha = - sync_openai_plugins_repo_via_git(tmp.path(), git_wrapper.to_str().expect("utf8 path")) - .expect("git sync should succeed"); - - assert_eq!(synced_sha, sha); - assert_curated_gmail_repo(&curated_plugins_repo_path(tmp.path())); - assert_eq!( - read_curated_plugins_sha(tmp.path()).as_deref(), - Some(sha.as_str()) - ); - assert!(!has_plugins_clone_dirs(tmp.path())); -} - -#[tokio::test] -async fn sync_openai_plugins_repo_falls_back_to_http_when_git_is_unavailable() { - let tmp = tempdir().expect("tempdir"); - let server = MockServer::start().await; - let sha = "0123456789abcdef0123456789abcdef01234567"; - - mount_github_repo_and_ref(&server, sha).await; - mount_github_zipball(&server, sha, curated_repo_zipball_bytes(sha)).await; - - let synced_sha = run_sync_with_transport_overrides( - tmp.path().to_path_buf(), - "missing-git-for-test", - server.uri(), - "http://127.0.0.1:9/backend-api/plugins/export/curated", - ) - .await - .expect("fallback sync should succeed"); - - let repo_path = curated_plugins_repo_path(tmp.path()); - assert_eq!(synced_sha, sha); - assert_curated_gmail_repo(&repo_path); - assert_eq!(read_curated_plugins_sha(tmp.path()).as_deref(), Some(sha)); -} - -#[cfg(unix)] -#[tokio::test] -async fn sync_openai_plugins_repo_falls_back_to_http_when_git_sync_fails() { - let tmp = tempdir().expect("tempdir"); - let bin_dir = tempfile::Builder::new() - .prefix("fake-git-fail-") - .tempdir() - .expect("tempdir"); - let git_path = bin_dir.path().join("git"); - let sha = "0123456789abcdef0123456789abcdef01234567"; - - write_executable_script( - &git_path, - r#"#!/bin/sh -echo "simulated git failure" >&2 -exit 1 -"#, - ); - - let server = MockServer::start().await; - mount_github_repo_and_ref(&server, sha).await; - mount_github_zipball(&server, sha, curated_repo_zipball_bytes(sha)).await; - - let synced_sha = run_sync_with_transport_overrides( - tmp.path().to_path_buf(), - git_path.to_str().expect("utf8 path"), - server.uri(), - "http://127.0.0.1:9/backend-api/plugins/export/curated", - ) - .await - .expect("fallback sync should succeed"); - - let repo_path = curated_plugins_repo_path(tmp.path()); - assert_eq!(synced_sha, sha); - assert_curated_gmail_repo(&repo_path); - assert_eq!(read_curated_plugins_sha(tmp.path()).as_deref(), Some(sha)); -} - -#[cfg(unix)] -#[test] -fn sync_openai_plugins_repo_via_git_cleans_up_staged_dir_on_clone_failure() { - let tmp = tempdir().expect("tempdir"); - let bin_dir = tempfile::Builder::new() - .prefix("fake-git-partial-fail-") - .tempdir() - .expect("tempdir"); - let git_path = bin_dir.path().join("git"); - let sha = "0123456789abcdef0123456789abcdef01234567"; - - write_executable_script( - &git_path, - &format!( - r#"#!/bin/sh -if [ "$1" = "ls-remote" ]; then - printf '%s\tHEAD\n' "{sha}" - exit 0 -fi -if [ "$1" = "clone" ]; then - dest="$5" - mkdir -p "$dest/.git" - echo "fatal: early EOF" >&2 - exit 128 -fi -echo "unexpected git invocation: $@" >&2 -exit 1 -"# - ), - ); - - let err = sync_openai_plugins_repo_via_git(tmp.path(), git_path.to_str().expect("utf8 path")) - .expect_err("git sync should fail"); - - assert!(err.contains("fatal: early EOF")); - assert!(!has_plugins_clone_dirs(tmp.path())); -} - -#[tokio::test] -async fn sync_openai_plugins_repo_via_http_cleans_up_staged_dir_on_extract_failure() { - let tmp = tempdir().expect("tempdir"); - let server = MockServer::start().await; - let sha = "0123456789abcdef0123456789abcdef01234567"; - - mount_github_repo_and_ref(&server, sha).await; - mount_github_zipball(&server, sha, b"not a zip archive".to_vec()).await; - - let err = run_http_sync(tmp.path().to_path_buf(), server.uri()) - .await - .expect_err("http sync should fail"); - - assert!(err.contains("failed to open curated plugins zip archive")); - assert!(!has_plugins_clone_dirs(tmp.path())); -} - -#[tokio::test] -async fn sync_openai_plugins_repo_skips_archive_download_when_sha_matches() { - let tmp = tempdir().expect("tempdir"); - let repo_path = curated_plugins_repo_path(tmp.path()); - std::fs::create_dir_all(repo_path.join(".agents/plugins")).expect("create repo"); - std::fs::write( - repo_path.join(".agents/plugins/marketplace.json"), - r#"{"name":"openai-curated","plugins":[]}"#, - ) - .expect("write marketplace"); - std::fs::create_dir_all(tmp.path().join(".tmp")).expect("create tmp"); - let sha = "fedcba9876543210fedcba9876543210fedcba98"; - std::fs::write(tmp.path().join(".tmp/plugins.sha"), format!("{sha}\n")).expect("write sha"); - - let server = MockServer::start().await; - mount_github_repo_and_ref(&server, sha).await; - - run_sync_with_transport_overrides( - tmp.path().to_path_buf(), - "missing-git-for-test", - server.uri(), - "http://127.0.0.1:9/backend-api/plugins/export/curated", - ) - .await - .expect("sync should succeed"); - - assert_eq!(read_curated_plugins_sha(tmp.path()).as_deref(), Some(sha)); - assert!(repo_path.join(".agents/plugins/marketplace.json").is_file()); -} - -#[tokio::test] -async fn sync_openai_plugins_repo_falls_back_to_export_archive_when_no_snapshot_exists() { - let tmp = tempdir().expect("tempdir"); - let server = MockServer::start().await; - let export_sha = "1111111111111111111111111111111111111111"; - - Mock::given(method("GET")) - .and(path("/repos/openai/plugins")) - .respond_with(ResponseTemplate::new(500).set_body_string("github repo lookup failed")) - .mount(&server) - .await; - let export_api_url = - mount_export_archive(&server, curated_repo_backup_archive_zip_bytes(export_sha)).await; - - let synced_sha = run_sync_with_transport_overrides( - tmp.path().to_path_buf(), - "missing-git-for-test", - server.uri(), - export_api_url, - ) - .await - .expect("export fallback sync should succeed"); - - let repo_path = curated_plugins_repo_path(tmp.path()); - assert_eq!(synced_sha, export_sha); - assert_curated_gmail_repo(&repo_path); - assert_eq!( - read_curated_plugins_sha(tmp.path()).as_deref(), - Some(export_sha) - ); -} - -#[tokio::test] -async fn sync_openai_plugins_repo_skips_export_archive_when_snapshot_exists() { - let tmp = tempdir().expect("tempdir"); - let curated_root = curated_plugins_repo_path(tmp.path()); - write_openai_curated_marketplace(&curated_root, &["linear"]); - write_curated_plugin_sha(tmp.path()); - - let plugin_manifest_path = curated_root.join("plugins/linear/.codex-plugin/plugin.json"); - let original_manifest = - std::fs::read_to_string(&plugin_manifest_path).expect("read existing plugin manifest"); - - let server = MockServer::start().await; - - Mock::given(method("GET")) - .and(path("/repos/openai/plugins")) - .respond_with(ResponseTemplate::new(500).set_body_string("github repo lookup failed")) - .mount(&server) - .await; - let export_api_url = mount_export_archive( - &server, - curated_repo_backup_archive_zip_bytes("2222222222222222222222222222222222222222"), - ) - .await; - - let err = run_sync_with_transport_overrides( - tmp.path().to_path_buf(), - "missing-git-for-test", - server.uri(), - export_api_url, - ) - .await - .expect_err("existing snapshot should suppress export fallback"); - - assert!(err.contains("export archive fallback skipped")); - assert_eq!( - std::fs::read_to_string(&plugin_manifest_path).expect("read plugin manifest after sync"), - original_manifest - ); - assert_eq!( - read_curated_plugins_sha(tmp.path()).as_deref(), - Some(TEST_CURATED_PLUGIN_SHA) - ); -} - -#[test] -fn read_extracted_backup_archive_git_sha_reads_head_ref_from_extracted_repo() { - let tmp = tempdir().expect("tempdir"); - let git_dir = tmp.path().join(".git/refs/heads"); - std::fs::create_dir_all(&git_dir).expect("create git ref dir"); - std::fs::write(tmp.path().join(".git/HEAD"), "ref: refs/heads/main\n").expect("write HEAD"); - std::fs::write( - git_dir.join("main"), - "3333333333333333333333333333333333333333\n", - ) - .expect("write main ref"); - - assert_eq!( - read_extracted_backup_archive_git_sha(tmp.path()) - .expect("read extracted backup archive git sha"), - Some("3333333333333333333333333333333333333333".to_string()) - ); -} - -#[test] -fn read_extracted_backup_archive_git_sha_rejects_non_refs_head_target() { - let tmp = tempdir().expect("tempdir"); - std::fs::create_dir_all(tmp.path().join(".git")).expect("create git dir"); - std::fs::write(tmp.path().join(".git/HEAD"), "ref: HEAD\n").expect("write HEAD"); - - let err = read_extracted_backup_archive_git_sha(tmp.path()) - .expect_err("non-refs target should be rejected"); - - assert!(err.contains("must stay under refs/")); -} - -#[test] -fn read_extracted_backup_archive_git_sha_rejects_path_traversal_ref() { - let tmp = tempdir().expect("tempdir"); - std::fs::create_dir_all(tmp.path().join(".git")).expect("create git dir"); - std::fs::write(tmp.path().join(".git/HEAD"), "ref: refs/heads/../../evil\n") - .expect("write HEAD"); - - let err = read_extracted_backup_archive_git_sha(tmp.path()) - .expect_err("path traversal ref should be rejected"); - - assert!(err.contains("invalid path components")); -} #[tokio::test] async fn startup_remote_plugin_sync_writes_marker_and_reconciles_state() { @@ -707,86 +88,3 @@ enabled = false let marker_contents = std::fs::read_to_string(marker_path).expect("marker should be readable"); assert_eq!(marker_contents, "ok\n"); } - -fn curated_repo_zipball_bytes(sha: &str) -> Vec { - let cursor = std::io::Cursor::new(Vec::new()); - let mut writer = ZipWriter::new(cursor); - let options = SimpleFileOptions::default(); - let root = format!("openai-plugins-{sha}"); - writer - .start_file(format!("{root}/.agents/plugins/marketplace.json"), options) - .expect("start marketplace entry"); - writer - .write_all( - br#"{ - "name": "openai-curated", - "plugins": [ - { - "name": "gmail", - "source": { - "source": "local", - "path": "./plugins/gmail" - } - } - ] -}"#, - ) - .expect("write marketplace"); - writer - .start_file( - format!("{root}/plugins/gmail/.codex-plugin/plugin.json"), - options, - ) - .expect("start plugin manifest entry"); - writer - .write_all(br#"{"name":"gmail"}"#) - .expect("write plugin manifest"); - - writer.finish().expect("finish zip writer").into_inner() -} - -fn curated_repo_backup_archive_zip_bytes(sha: &str) -> Vec { - let cursor = std::io::Cursor::new(Vec::new()); - let mut writer = ZipWriter::new(cursor); - let options = SimpleFileOptions::default(); - - writer - .start_file("plugins/.git/HEAD", options) - .expect("start HEAD entry"); - writer - .write_all(b"ref: refs/heads/main\n") - .expect("write HEAD"); - writer - .start_file("plugins/.git/refs/heads/main", options) - .expect("start main ref entry"); - writer - .write_all(format!("{sha}\n").as_bytes()) - .expect("write main ref"); - writer - .start_file("plugins/.agents/plugins/marketplace.json", options) - .expect("start marketplace entry"); - writer - .write_all( - br#"{ - "name": "openai-curated", - "plugins": [ - { - "name": "gmail", - "source": { - "source": "local", - "path": "./plugins/gmail" - } - } - ] -}"#, - ) - .expect("write marketplace"); - writer - .start_file("plugins/plugins/gmail/.codex-plugin/plugin.json", options) - .expect("start plugin manifest entry"); - writer - .write_all(br#"{"name":"gmail"}"#) - .expect("write plugin manifest"); - - writer.finish().expect("finish zip writer").into_inner() -} diff --git a/codex-rs/core/src/plugins/test_support.rs b/codex-rs/core/src/plugins/test_support.rs index 8624d40810..8fbaebb803 100644 --- a/codex-rs/core/src/plugins/test_support.rs +++ b/codex-rs/core/src/plugins/test_support.rs @@ -3,7 +3,7 @@ use crate::config::ConfigBuilder; use std::fs; use std::path::Path; -use super::OPENAI_CURATED_MARKETPLACE_NAME; +use codex_core_plugins::OPENAI_CURATED_MARKETPLACE_NAME; pub(crate) const TEST_CURATED_PLUGIN_SHA: &str = "0123456789abcdef0123456789abcdef01234567"; diff --git a/codex-rs/core/src/tools/handlers/tool_suggest_tests.rs b/codex-rs/core/src/tools/handlers/tool_suggest_tests.rs index 77ab793dc0..5290d113f5 100644 --- a/codex-rs/core/src/tools/handlers/tool_suggest_tests.rs +++ b/codex-rs/core/src/tools/handlers/tool_suggest_tests.rs @@ -5,13 +5,14 @@ use crate::plugins::test_support::load_plugins_config; use crate::plugins::test_support::write_curated_plugin_sha; use crate::plugins::test_support::write_openai_curated_marketplace; use crate::plugins::test_support::write_plugins_feature_config; +use codex_core_plugins::startup_sync::curated_plugins_repo_path; use codex_utils_absolute_path::AbsolutePathBuf; use tempfile::tempdir; #[tokio::test] async fn verified_plugin_suggestion_completed_requires_installed_plugin() { let codex_home = tempdir().expect("tempdir should succeed"); - let curated_root = crate::plugins::curated_plugins_repo_path(codex_home.path()); + let curated_root = curated_plugins_repo_path(codex_home.path()); write_openai_curated_marketplace(&curated_root, &["sample"]); write_curated_plugin_sha(codex_home.path()); write_plugins_feature_config(codex_home.path()); diff --git a/codex-rs/tui/Cargo.toml b/codex-rs/tui/Cargo.toml index 29d6d0c393..300449a414 100644 --- a/codex-rs/tui/Cargo.toml +++ b/codex-rs/tui/Cargo.toml @@ -34,6 +34,7 @@ codex-chatgpt = { workspace = true } codex-cloud-requirements = { workspace = true } codex-config = { workspace = true } codex-connectors = { workspace = true } +codex-core-plugins = { workspace = true } codex-core-skills = { workspace = true } codex-exec-server = { workspace = true } codex-features = { workspace = true } diff --git a/codex-rs/tui/src/chatwidget/plugins.rs b/codex-rs/tui/src/chatwidget/plugins.rs index f330a80f22..ae6b08c07d 100644 --- a/codex-rs/tui/src/chatwidget/plugins.rs +++ b/codex-rs/tui/src/chatwidget/plugins.rs @@ -12,7 +12,6 @@ use crate::bottom_pane::SelectionTab; use crate::bottom_pane::SelectionToggle; use crate::bottom_pane::SelectionViewParams; use crate::history_cell; -use crate::legacy_core::plugins::OPENAI_CURATED_MARKETPLACE_NAME; use crate::onboarding::mark_url_hyperlink; use crate::render::renderable::ColumnRenderable; use crate::render::renderable::Renderable; @@ -26,6 +25,7 @@ use codex_app_server_protocol::PluginMarketplaceEntry; use codex_app_server_protocol::PluginReadResponse; use codex_app_server_protocol::PluginSummary; use codex_app_server_protocol::PluginUninstallResponse; +use codex_core_plugins::OPENAI_CURATED_MARKETPLACE_NAME; use codex_features::Feature; use codex_utils_absolute_path::AbsolutePathBuf; use ratatui::buffer::Buffer; diff --git a/codex-rs/tui/src/chatwidget/tests.rs b/codex-rs/tui/src/chatwidget/tests.rs index 487e8a3565..f4f7dede2a 100644 --- a/codex-rs/tui/src/chatwidget/tests.rs +++ b/codex-rs/tui/src/chatwidget/tests.rs @@ -19,7 +19,6 @@ pub(super) use crate::legacy_core::config::Config; pub(super) use crate::legacy_core::config::ConfigBuilder; pub(super) use crate::legacy_core::config::Constrained; pub(super) use crate::legacy_core::config::ConstraintError; -pub(super) use crate::legacy_core::plugins::OPENAI_CURATED_MARKETPLACE_NAME; pub(super) use crate::model_catalog::ModelCatalog; pub(super) use crate::test_backend::VT100Backend; pub(super) use crate::test_support::PathBufExt; @@ -106,6 +105,7 @@ pub(super) use codex_config::types::ApprovalsReviewer; pub(super) use codex_config::types::Notifications; #[cfg(target_os = "windows")] pub(super) use codex_config::types::WindowsSandboxModeToml; +pub(super) use codex_core_plugins::OPENAI_CURATED_MARKETPLACE_NAME; pub(super) use codex_core_skills::model::SkillMetadata; pub(super) use codex_features::FEATURES; pub(super) use codex_features::Feature;