mirror of
https://github.com/openai/codex.git
synced 2026-04-30 03:12:20 +03:00
343d1af3da
## Why This PR is the current, consolidated follow-up to the earlier Windows Bazel attempt in #11229. The goal is no longer just to get a tiny Windows smoke job limping along: it is to make the ordinary Bazel CI path usable on `windows-latest` for `x86_64-pc-windows-gnullvm`, with the same broad `//...` test shape that macOS and Linux already use. The earlier smoke-list version of this work was useful as a foothold, but it was not a good long-term landing point. Windows Bazel kept surfacing real issues outside that allowlist: - GitHub's Windows runner exposed runfiles-manifest bugs such as `FINDSTR: Cannot open D:MANIFEST`, which broke Bazel test launchers even when the manifest file existed. - `rules_rs`, `rules_rust`, LLVM extraction, and Abseil still needed `windows-gnullvm`-specific fixes for our hermetic toolchain. - the V8 path needed more work than just turning the Windows matrix entry back on: `rusty_v8` does not ship Windows GNU artifacts in the same shape we need, and Bazel's in-tree V8 build needed a set of Windows GNU portability fixes. Windows performance pressure also pushed this toward a full solution instead of a permanent smoke suite. During this investigation we hit targets such as `//codex-rs/shell-command:shell-command-unit-tests` that were much more expensive on Windows because they repeatedly spawn real PowerShell parsers (see #16057 for one concrete example of that pressure). That made it much more valuable to get the real Windows Bazel path working than to keep iterating on a narrowly curated subset. The net result is that this PR now aims for the same CI contract on Windows that we already expect elsewhere: keep standalone `//third_party/v8:all` out of the ordinary Bazel lane, but allow V8 consumers under `//codex-rs/...` to build and test transitively through `//...`. ## What Changed ### CI and workflow wiring - re-enable the `windows-latest` / `x86_64-pc-windows-gnullvm` Bazel matrix entry in `.github/workflows/bazel.yml` - move the Windows Bazel output root to `D:\b` and enable `git config --global core.longpaths true` in `.github/actions/setup-bazel-ci/action.yml` - keep the ordinary Bazel target set on Windows aligned with macOS and Linux by running `//...` while excluding only standalone `//third_party/v8:all` targets from the normal lane ### Toolchain and module support for `windows-gnullvm` - patch `rules_rs` so `windows-gnullvm` is modeled as a distinct Windows exec/toolchain platform instead of collapsing into the generic Windows shape - patch `rules_rust` build-script environment handling so llvm-mingw build-script probes do not inherit unsupported `-fstack-protector*` flags - patch the LLVM module archive so it extracts cleanly on Windows and provides the MinGW libraries this toolchain needs - patch Abseil so its thread-local identity path matches the hermetic `windows-gnullvm` toolchain instead of taking an incompatible MinGW pthread path - keep both MSVC and GNU Windows targets in the generated Cargo metadata because the current V8 release-asset story still uses MSVC-shaped names in some places while the Bazel build targets the GNU ABI ### Windows test-launch and binary-behavior fixes - update `workspace_root_test_launcher.bat.tpl` to read the runfiles manifest directly instead of shelling out to `findstr`, which was the source of the `D:MANIFEST` failures on the GitHub Windows runner - thread a larger Windows GNU stack reserve through `defs.bzl` so Bazel-built binaries that pull in V8 behave correctly both under normal builds and under `bazel test` - remove the no-longer-needed Windows bootstrap sh-toolchain override from `.bazelrc` ### V8 / `rusty_v8` Windows GNU support - export and apply the new Windows GNU patch set from `patches/BUILD.bazel` / `MODULE.bazel` - patch the V8 module/rules/source layers so the in-tree V8 build can produce Windows GNU archives under Bazel - teach `third_party/v8/BUILD.bazel` to build Windows GNU static archives in-tree instead of aliasing them to the MSVC prebuilts - reuse the Linux release binding for the experimental Windows GNU path where `rusty_v8` does not currently publish a Windows GNU binding artifact ## Testing - the primary end-to-end validation for this work is the `Bazel` workflow plus `v8-canary`, since the hard parts are Windows-specific and depend on real GitHub runner behavior - before consolidation back onto this PR, the same net change passed the full Bazel matrix in [run 23675590471](https://github.com/openai/codex/actions/runs/23675590471) and passed `v8-canary` in [run 23675590453](https://github.com/openai/codex/actions/runs/23675590453) - those successful runs included the `windows-latest` / `x86_64-pc-windows-gnullvm` Bazel job with the ordinary `//...` path, not the earlier Windows smoke allowlist --- [//]: # (BEGIN SAPLING FOOTER) Stack created with [Sapling](https://sapling-scm.com). Best reviewed with [ReviewStack](https://reviewstack.dev/openai/codex/pull/15952). * #16067 * __->__ #15952
270 lines
11 KiB
Diff
270 lines
11 KiB
Diff
# What: replace upstream V8 module dependency bootstrapping with repository
|
|
# declarations and dependency setup that match this Bazel workspace.
|
|
# Scope: upstream MODULE.bazel only; affects external repo resolution and Bazel
|
|
# module wiring, not V8 source files.
|
|
|
|
diff --git a/orig/v8-14.6.202.11/MODULE.bazel b/mod/v8-14.6.202.11/MODULE.bazel
|
|
--- a/orig/v8-14.6.202.11/MODULE.bazel
|
|
+++ b/mod/v8-14.6.202.11/MODULE.bazel
|
|
@@ -8,7 +8,57 @@
|
|
bazel_dep(name = "rules_python", version = "1.0.0")
|
|
bazel_dep(name = "platforms", version = "1.0.0")
|
|
bazel_dep(name = "abseil-cpp", version = "20250814.0")
|
|
-bazel_dep(name = "highway", version = "1.2.0")
|
|
+bazel_dep(name = "rules_license", version = "0.0.4")
|
|
+
|
|
+git_repository = use_repo_rule("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
|
|
+http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
|
|
+
|
|
+http_archive(
|
|
+ name = "highway",
|
|
+ patch_args = ["-p1"],
|
|
+ patches = ["@v8//:bazel/highway.patch"],
|
|
+ sha256 = "7e0be78b8318e8bdbf6fa545d2ecb4c90f947df03f7aadc42c1967f019e63343",
|
|
+ strip_prefix = "highway-1.2.0",
|
|
+ urls = ["https://github.com/google/highway/archive/refs/tags/1.2.0.tar.gz"],
|
|
+)
|
|
+
|
|
+git_repository(
|
|
+ name = "icu",
|
|
+ build_file = "@v8//:bazel/BUILD.icu",
|
|
+ commit = "a86a32e67b8d1384b33f8fa48c83a6079b86f8cd",
|
|
+ patch_cmds = ["find source -name BUILD.bazel | xargs rm"],
|
|
+ patch_cmds_win = ["Get-ChildItem -Path source -File -Include BUILD.bazel -Recurse | Remove-Item"],
|
|
+ remote = "https://chromium.googlesource.com/chromium/deps/icu.git",
|
|
+)
|
|
+
|
|
+http_archive(
|
|
+ name = "fast_float",
|
|
+ build_file_content = 'load("@rules_cc//cc:defs.bzl", "cc_library")\n\ncc_library(\n name = "fast_float",\n hdrs = glob(["include/fast_float/*.h"]),\n include_prefix = "third_party/fast_float/src/include",\n strip_include_prefix = "include",\n visibility = ["//visibility:public"],\n)\n',
|
|
+ sha256 = "e14a33089712b681d74d94e2a11362643bd7d769ae8f7e7caefe955f57f7eacd",
|
|
+ strip_prefix = "fast_float-8.0.2",
|
|
+ urls = ["https://github.com/fastfloat/fast_float/archive/refs/tags/v8.0.2.tar.gz"],
|
|
+)
|
|
+
|
|
+git_repository(
|
|
+ name = "simdutf",
|
|
+ build_file_content = 'load("@rules_cc//cc:defs.bzl", "cc_library")\n\ncc_library(\n name = "simdutf",\n srcs = ["simdutf.cpp"],\n hdrs = ["simdutf.h"],\n copts = ["-std=c++20"],\n include_prefix = "third_party/simdutf",\n visibility = ["//visibility:public"],\n)\n',
|
|
+ commit = "93b35aec29256f705c97f675fe4623578bd7a395",
|
|
+ remote = "https://chromium.googlesource.com/chromium/src/third_party/simdutf",
|
|
+)
|
|
+
|
|
+git_repository(
|
|
+ name = "dragonbox",
|
|
+ build_file_content = 'load("@rules_cc//cc:defs.bzl", "cc_library")\n\ncc_library(\n name = "dragonbox",\n hdrs = ["include/dragonbox/dragonbox.h"],\n include_prefix = "third_party/dragonbox/src/include",\n strip_include_prefix = "include",\n visibility = ["//visibility:public"],\n)\n',
|
|
+ commit = "beeeef91cf6fef89a4d4ba5e95d47ca64ccb3a44",
|
|
+ remote = "https://chromium.googlesource.com/external/github.com/jk-jeon/dragonbox.git",
|
|
+)
|
|
+
|
|
+git_repository(
|
|
+ name = "fp16",
|
|
+ build_file_content = 'load("@rules_cc//cc:defs.bzl", "cc_library")\n\ncc_library(\n name = "fp16",\n hdrs = glob(["include/**/*.h"]),\n include_prefix = "third_party/fp16/src/include",\n includes = ["include"],\n strip_include_prefix = "include",\n visibility = ["//visibility:public"],\n)\n',
|
|
+ commit = "3d2de1816307bac63c16a297e8c4dc501b4076df",
|
|
+ remote = "https://chromium.googlesource.com/external/github.com/Maratyszcza/FP16.git",
|
|
+)
|
|
|
|
pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
|
|
pip.parse(
|
|
@@ -22,171 +72,3 @@
|
|
)
|
|
use_repo(pip, "v8_python_deps")
|
|
|
|
-# Define the local LLVM toolchain repository
|
|
-llvm_toolchain_repository = use_repo_rule("//bazel/toolchain:llvm_repository.bzl", "llvm_toolchain_repository")
|
|
-
|
|
-llvm_toolchain_repository(
|
|
- name = "llvm_toolchain",
|
|
- path = "third_party/llvm-build/Release+Asserts",
|
|
- config_file_content = """
|
|
-load("@bazel_tools//tools/cpp:cc_toolchain_config_lib.bzl", "feature", "flag_group", "flag_set", "tool_path")
|
|
-
|
|
-def _impl(ctx):
|
|
- tool_paths = [
|
|
- tool_path(name = "gcc", path = "bin/clang"),
|
|
- tool_path(name = "ld", path = "bin/lld"),
|
|
- tool_path(name = "ar", path = "bin/llvm-ar"),
|
|
- tool_path(name = "cpp", path = "bin/clang++"),
|
|
- tool_path(name = "gcov", path = "/bin/false"),
|
|
- tool_path(name = "nm", path = "bin/llvm-nm"),
|
|
- tool_path(name = "objdump", path = "bin/llvm-objdump"),
|
|
- tool_path(name = "strip", path = "bin/llvm-strip"),
|
|
- ]
|
|
-
|
|
- features = [
|
|
- feature(
|
|
- name = "default_compile_flags",
|
|
- enabled = True,
|
|
- flag_sets = [
|
|
- flag_set(
|
|
- actions = [
|
|
- "c-compile",
|
|
- "c++-compile",
|
|
- "c++-header-parsing",
|
|
- "c++-module-compile",
|
|
- "c++-module-codegen",
|
|
- "linkstamp-compile",
|
|
- "assemble",
|
|
- "preprocess-assemble",
|
|
- ],
|
|
- flag_groups = [
|
|
- flag_group(
|
|
- flags = [
|
|
- "--sysroot={WORKSPACE_ROOT}/build/linux/debian_bullseye_amd64-sysroot",
|
|
- "-nostdinc++",
|
|
- "-isystem",
|
|
- "{WORKSPACE_ROOT}/buildtools/third_party/libc++",
|
|
- "-isystem",
|
|
- "{WORKSPACE_ROOT}/third_party/libc++/src/include",
|
|
- "-isystem",
|
|
- "{WORKSPACE_ROOT}/third_party/libc++abi/src/include",
|
|
- "-isystem",
|
|
- "{WORKSPACE_ROOT}/third_party/libc++/src/src",
|
|
- "-isystem",
|
|
- "{WORKSPACE_ROOT}/third_party/llvm-libc/src",
|
|
- "-D_LIBCPP_HARDENING_MODE_DEFAULT=_LIBCPP_HARDENING_MODE_NONE",
|
|
- "-DLIBC_NAMESPACE=__llvm_libc_cr",
|
|
- ],
|
|
- ),
|
|
- ],
|
|
- ),
|
|
- ],
|
|
- ),
|
|
- feature(
|
|
- name = "default_linker_flags",
|
|
- enabled = True,
|
|
- flag_sets = [
|
|
- flag_set(
|
|
- actions = [
|
|
- "c++-link-executable",
|
|
- "c++-link-dynamic-library",
|
|
- "c++-link-nodeps-dynamic-library",
|
|
- ],
|
|
- flag_groups = [
|
|
- flag_group(
|
|
- flags = [
|
|
- "--sysroot={WORKSPACE_ROOT}/build/linux/debian_bullseye_amd64-sysroot",
|
|
- "-fuse-ld=lld",
|
|
- "-lm",
|
|
- "-lpthread",
|
|
- ],
|
|
- ),
|
|
- ],
|
|
- ),
|
|
- ],
|
|
- ),
|
|
- ]
|
|
-
|
|
- return cc_common.create_cc_toolchain_config_info(
|
|
- ctx = ctx,
|
|
- features = features,
|
|
- cxx_builtin_include_directories = [
|
|
- "{WORKSPACE_ROOT}/buildtools/third_party/libc++",
|
|
- "{WORKSPACE_ROOT}/third_party/libc++/src/include",
|
|
- "{WORKSPACE_ROOT}/third_party/libc++abi/src/include",
|
|
- "{WORKSPACE_ROOT}/third_party/libc++/src/src",
|
|
- "{WORKSPACE_ROOT}/third_party/llvm-libc/src",
|
|
- "{WORKSPACE_ROOT}/third_party/llvm-build/Release+Asserts/lib/clang/22/include",
|
|
- "{WORKSPACE_ROOT}/third_party/llvm-build/Release+Asserts/lib/clang/23/include",
|
|
- "{WORKSPACE_ROOT}/build/linux/debian_bullseye_amd64-sysroot/usr/include",
|
|
- "{WORKSPACE_ROOT}/build/linux/debian_bullseye_amd64-sysroot/usr/local/include",
|
|
- ],
|
|
- toolchain_identifier = "local_clang",
|
|
- host_system_name = "local",
|
|
- target_system_name = "local",
|
|
- target_cpu = "k8",
|
|
- target_libc = "unknown",
|
|
- compiler = "clang",
|
|
- abi_version = "unknown",
|
|
- abi_libc_version = "unknown",
|
|
- tool_paths = tool_paths,
|
|
- )
|
|
-
|
|
-cc_toolchain_config = rule(
|
|
- implementation = _impl,
|
|
- attrs = {},
|
|
- provides = [CcToolchainConfigInfo],
|
|
-)
|
|
-""",
|
|
- build_file_content = """
|
|
-load(":cc_toolchain_config.bzl", "cc_toolchain_config")
|
|
-
|
|
-package(default_visibility = ["//visibility:public"])
|
|
-
|
|
-filegroup(
|
|
- name = "all_files",
|
|
- srcs = glob(["**/*"]),
|
|
-)
|
|
-
|
|
-filegroup(name = "empty")
|
|
-
|
|
-cc_toolchain_config(name = "k8_toolchain_config")
|
|
-
|
|
-cc_toolchain(
|
|
- name = "k8_toolchain",
|
|
- all_files = ":all_files",
|
|
- ar_files = ":all_files",
|
|
- compiler_files = ":all_files",
|
|
- dwp_files = ":empty",
|
|
- linker_files = ":all_files",
|
|
- objcopy_files = ":all_files",
|
|
- strip_files = ":all_files",
|
|
- supports_param_files = 0,
|
|
- toolchain_config = ":k8_toolchain_config",
|
|
- toolchain_identifier = "local_clang",
|
|
-)
|
|
-
|
|
-toolchain(
|
|
- name = "cc_toolchain_k8",
|
|
- exec_compatible_with = [
|
|
- "@platforms//cpu:x86_64",
|
|
- "@platforms//os:linux",
|
|
- ],
|
|
- target_compatible_with = [
|
|
- "@platforms//cpu:x86_64",
|
|
- "@platforms//os:linux",
|
|
- ],
|
|
- toolchain = ":k8_toolchain",
|
|
- toolchain_type = "@bazel_tools//tools/cpp:toolchain_type",
|
|
-)
|
|
-""",
|
|
-)
|
|
-
|
|
-register_toolchains("@llvm_toolchain//:cc_toolchain_k8")
|
|
-
|
|
-# Define local repository for libc++ from third_party sources
|
|
-libcxx_repository = use_repo_rule("//bazel/toolchain:libcxx_repository.bzl", "libcxx_repository")
|
|
-
|
|
-libcxx_repository(
|
|
- name = "libcxx",
|
|
-)
|
|
diff --git a/orig/v8-14.6.202.11/bazel/highway.patch b/mod/v8-14.6.202.11/bazel/highway.patch
|
|
new file mode 100644
|
|
--- /dev/null
|
|
+++ b/mod/v8-14.6.202.11/bazel/highway.patch
|
|
@@ -0,0 +1,25 @@
|
|
+diff --git a/BUILD b/BUILD
|
|
+--- a/BUILD
|
|
++++ b/BUILD
|
|
+@@ -2,7 +2,7 @@
|
|
+ load("@bazel_skylib//lib:selects.bzl", "selects")
|
|
+ load("@rules_license//rules:license.bzl", "license")
|
|
+
|
|
+-load("@rules_cc//cc:defs.bzl", "cc_test")
|
|
++load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test")
|
|
+ # Placeholder#2 for Guitar, do not remove
|
|
+
|
|
+ package(
|
|
+@@ -134,11 +134,7 @@ COPTS = select({
|
|
+ ],
|
|
+ })
|
|
+
|
|
+-DEFINES = select({
|
|
+- ":compiler_msvc": ["HWY_SHARED_DEFINE"],
|
|
+- ":compiler_clangcl": ["HWY_SHARED_DEFINE"],
|
|
+- "//conditions:default": [],
|
|
+-})
|
|
++DEFINES = []
|
|
+
|
|
+ # Unused on Bazel builds, where this is not defined/known; Copybara replaces
|
|
+ # usages with an empty list.
|