From 61af2417a9cb6c444008a5a8d386e784bd7d5d8a Mon Sep 17 00:00:00 2001 From: kp2pml30 Date: Fri, 16 Oct 2026 22:35:03 +0900 Subject: [PATCH 1/2] =?UTF-8?q?fix(ci):=20build=20releases=20at=20the=20ov?= =?UTF-8?q?erridden=20version=20=F0=9F=90=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A release dispatched with version_override only renamed the tag: every cell still built from .genvm-monorepo-root, so the binaries reported the old version. The plan now hands its tag to every build and test cell, and apply-release-version writes it into .genvm-monorepo-root before nix runs. An override therefore misses the nix cache entirely. --- .../workflows/incl_release_build_test.yaml | 4 +- .../incl_release_build_test_cell_build.yaml | 9 ++ .../incl_release_build_test_cell_test.yaml | 9 ++ .github/workflows/release.yaml | 2 +- .../howto/releasing/release-build.md | 6 ++ support/ci/pipelines/release.py | 45 +++++++++- support/ci/unit_tests/test_release.py | 84 +++++++++++++++++++ 7 files changed, 153 insertions(+), 6 deletions(-) create mode 100644 support/ci/unit_tests/test_release.py diff --git a/.github/workflows/incl_release_build_test.yaml b/.github/workflows/incl_release_build_test.yaml index 46c85933..c46ac8af 100644 --- a/.github/workflows/incl_release_build_test.yaml +++ b/.github/workflows/incl_release_build_test.yaml @@ -13,7 +13,7 @@ on: workflow_call: inputs: version_override: - description: "Release tag. Defaults to `version` from .genvm-monorepo-root" + description: "Release tag and embedded version. Defaults to `version` from .genvm-monorepo-root" required: false type: string default: "" @@ -117,6 +117,7 @@ jobs: runs_on: ${{ matrix.runs_on }} artifact_name: ${{ matrix.artifact_name }} job_json: ${{ matrix.job_json }} + version_override: ${{ needs.plan.outputs.tag }} secrets: inherit test: @@ -134,4 +135,5 @@ jobs: with_nix: ${{ matrix.with_nix }} setup_python: ${{ matrix.setup_python }} job_json: ${{ matrix.job_json }} + version_override: ${{ needs.plan.outputs.tag }} secrets: inherit diff --git a/.github/workflows/incl_release_build_test_cell_build.yaml b/.github/workflows/incl_release_build_test_cell_build.yaml index 0b25bc5c..77359317 100644 --- a/.github/workflows/incl_release_build_test_cell_build.yaml +++ b/.github/workflows/incl_release_build_test_cell_build.yaml @@ -11,6 +11,10 @@ name: Build one release artifact cell on: workflow_call: inputs: + version_override: + description: release version from the plan + required: true + type: string name: required: true type: string @@ -41,6 +45,11 @@ jobs: steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - name: Apply release version + env: + VERSION_OVERRIDE: ${{ inputs.version_override }} + run: ./support/ci/run.sh pipeline apply-release-version --version-override "$VERSION_OVERRIDE" + # Guards the "Job:" step below: every key must have a step, every step a key. - name: Check job keys env: diff --git a/.github/workflows/incl_release_build_test_cell_test.yaml b/.github/workflows/incl_release_build_test_cell_test.yaml index 1ac09490..aa5adab2 100644 --- a/.github/workflows/incl_release_build_test_cell_test.yaml +++ b/.github/workflows/incl_release_build_test_cell_test.yaml @@ -8,6 +8,10 @@ name: Test one release platform cell on: workflow_call: inputs: + version_override: + description: release version from the plan + required: true + type: string name: required: true type: string @@ -49,6 +53,11 @@ jobs: steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - name: Apply release version + env: + VERSION_OVERRIDE: ${{ inputs.version_override }} + run: ./support/ci/run.sh pipeline apply-release-version --version-override "$VERSION_OVERRIDE" + - name: Check job keys env: JOB_JSON: ${{ inputs.job_json }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index fbbf6ecd..6e137912 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -24,7 +24,7 @@ on: version_override: type: string required: false - description: "Override release tag (e.g., v1.2.3). Defaults to `version` from .genvm-monorepo-root" + description: "Override release tag and embedded version (e.g., v1.2.3). Defaults to `version` from .genvm-monorepo-root" defaults: run: diff --git a/docs/contributing/howto/releasing/release-build.md b/docs/contributing/howto/releasing/release-build.md index 714f1739..d47e6319 100644 --- a/docs/contributing/howto/releasing/release-build.md +++ b/docs/contributing/howto/releasing/release-build.md @@ -39,6 +39,12 @@ one install root, so a full install is one platform asset plus `genvm-universal`. The platform asset already carries every active executor line ([versioning.md](versioning.md)) +`version_override` sets both the release tag and the embedded manager version. +The plan passes the resolved version to every build and test cell, which applies +it to `.genvm-monorepo-root` before source setup. Nix reads that file for +`GENVM_PROFILE`, so an override changes the build inputs and cache keys. +Executor versions remain pinned by their own manifests + ## Verifying an Asset The publisher uploads `SHA256SUMS` next to the tarballs and records a keyless diff --git a/support/ci/pipelines/release.py b/support/ci/pipelines/release.py index aa7dc837..839d5148 100644 --- a/support/ci/pipelines/release.py +++ b/support/ci/pipelines/release.py @@ -2,6 +2,7 @@ import json import os import re +import subprocess import ci_lib from pipelines.tests import GithubStep, SubStep, step_nix_develop, step_run @@ -238,6 +239,43 @@ def test_cells() -> list[dict]: return [_cell_amd64_linux(), _cell_arm64_linux(), _cell_arm64_macos()] +def check_release_version(version: str) -> bool: + ident = r'[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*' + semver = re.fullmatch(rf'v[0-9]+\.[0-9]+\.[0-9]+(-{ident})?(\+{ident})?', version) + if ( + not semver + or subprocess.run(['git', 'check-ref-format', f'refs/tags/{version}']).returncode + != 0 + ): + ci_lib.github_error(f'invalid version {version!r}') + return False + return True + + +class ApplyReleaseVersion(ci_lib.Pipeline): + def name(self) -> str: + return 'apply-release-version' + + def add_to(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument( + '--version-override', + default='', + help='release version to write to .genvm-monorepo-root; empty leaves it unchanged', + ) + + def handler(self, args: argparse.Namespace) -> int: + if not args.version_override: + return 0 + if not check_release_version(args.version_override): + return 1 + path = ci_lib.ROOT_DIR / '.genvm-monorepo-root' + monorepo = json.loads(path.read_text()) + if monorepo['version'] != args.version_override: + monorepo['version'] = args.version_override + path.write_text(json.dumps(monorepo, indent=4) + '\n') + return 0 + + class PlanReleaseMatrix(ci_lib.Pipeline): """ Emit the release build/test matrices and the tag they are built at. @@ -253,7 +291,7 @@ def add_to(self, parser: argparse.ArgumentParser) -> None: parser.add_argument( '--version-override', default='', - help='release tag; defaults to `version` from .genvm-monorepo-root', + help='release tag and embedded version; defaults to `version` from .genvm-monorepo-root', ) def handler(self, args: argparse.Namespace) -> int: @@ -263,8 +301,7 @@ def handler(self, args: argparse.Namespace) -> int: # Shape only. Whether the tag is already taken depends on the remote and # is the publisher's business — a PR-label run plans the same tag every # time and must not care. - if not re.match(r'^v[0-9]+\.[0-9]+\.[0-9]+', tag): - ci_lib.github_error(f'invalid version {tag!r}') + if not check_release_version(tag): return 1 builds = build_cells() @@ -301,4 +338,4 @@ def handler(self, args: argparse.Namespace) -> int: return 0 -COMMANDS = [PlanReleaseMatrix()] +COMMANDS = [PlanReleaseMatrix(), ApplyReleaseVersion()] diff --git a/support/ci/unit_tests/test_release.py b/support/ci/unit_tests/test_release.py new file mode 100644 index 00000000..d4b714ed --- /dev/null +++ b/support/ci/unit_tests/test_release.py @@ -0,0 +1,84 @@ +import argparse +import json + +import pytest + +from pipelines import release + + +@pytest.fixture +def monorepo(tmp_path, monkeypatch): + path = tmp_path / '.genvm-monorepo-root' + path.write_text( + json.dumps( + { + 'version': 'v0.6.0-rc2', + 'active-versions': ['v0.3', 'v0.2'], + 'support-only-versions': ['v0.2'], + 'artifacts_dir': 'build/test-artifacts', + }, + indent=4, + ) + + '\n' + ) + monkeypatch.setattr(release.ci_lib, 'ROOT_DIR', tmp_path) + monkeypatch.setattr(release, 'release_filter_tag', lambda: 'integration & stable') + return path + + +def run(command, override=None): + parser = argparse.ArgumentParser() + command.add_to(parser) + argv = [] if override is None else ['--version-override', override] + return command.handler(parser.parse_args(argv)) + + +@pytest.mark.parametrize('override', [None, '', 'v0.6.0-rc2']) +def test_apply_unchanged_version_does_not_write(monorepo, override): + before = monorepo.read_bytes() + mtime = monorepo.stat().st_mtime_ns + assert run(release.ApplyReleaseVersion(), override) == 0 + assert monorepo.read_bytes() == before + assert monorepo.stat().st_mtime_ns == mtime + + +@pytest.mark.parametrize('override', ['v1.2.3', 'v0.6.0-rc3', 'v1.2.3+build.4']) +def test_apply_changes_only_version_and_preserves_format(monorepo, override): + before = monorepo.read_text() + assert run(release.ApplyReleaseVersion(), override) == 0 + assert monorepo.read_text() == before.replace('v0.6.0-rc2', override) + + +@pytest.mark.parametrize( + 'override', + [ + '1.2.3', + 'v1.2', + 'release', + ' v1.2.3', + 'v1.2.3 rc', + 'v1.2.3;x', + 'v1.2.3-rc..1', + 'v1.2.3-rc.', + 'v1.2.3-x.lock', + ], +) +def test_plan_and_apply_reject_invalid_version( + monorepo, tmp_path, monkeypatch, override +): + before = monorepo.read_bytes() + output = tmp_path / 'github-output' + monkeypatch.setenv('GITHUB_OUTPUT', str(output)) + assert run(release.PlanReleaseMatrix(), override) == 1 + assert run(release.ApplyReleaseVersion(), override) == 1 + assert monorepo.read_bytes() == before + assert not output.exists() + + +@pytest.mark.parametrize('override', ['', 'v1.2.3-rc1']) +def test_plan_tags_with_override(monorepo, tmp_path, monkeypatch, override): + output = tmp_path / 'github-output' + monkeypatch.setenv('GITHUB_OUTPUT', str(output)) + assert run(release.PlanReleaseMatrix(), override) == 0 + outputs = dict(line.split('=', 1) for line in output.read_text().splitlines()) + assert outputs['tag'] == (override or 'v0.6.0-rc2') From 50d1ec162f6aead514fddc13daedcad917e66386 Mon Sep 17 00:00:00 2001 From: kp2pml30 Date: Fri, 16 Oct 2026 22:35:52 +0900 Subject: [PATCH 2/2] =?UTF-8?q?perf(ci):=20shrink=20the=20Rust=20toolchain?= =?UTF-8?q?=20release=20tests=20download=20=E2=9A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(nix): stop genvm-rust from storing every component twice 🐛 * perf(ci): run release tests in a slim release-tests dev shell ⚡ * fix(nix): keep zig out of genvm-rust built without the zig wrapper 🐛 rust.nix copied each tarball's top-level directory as well as the component directories inside it, so $out held every component twice. The dev toolchain shrinks from 2.96 to 1.45 GiB of NAR, and the builder toolchain from 2.17 to 1.06 GiB. The amd64-linux release test cell used the mock-tests shell, which pulled 2 toolchains plus a from-source cargo-afl build: about 14 min of its 28. The release-tests shell carries only cargo, rustc, the host and wasm32-wasip1 std (0.74 GiB closure, zig no longer leaks in through the rpath), wabt and the runner's Python deps. --- flake.nix | 17 +++++++++++++++++ support/ci/pipelines/release.py | 2 +- support/rust.nix | 25 +++++++++++++++---------- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/flake.nix b/flake.nix index 6f4f854b..d8a7bef0 100644 --- a/flake.nix +++ b/flake.nix @@ -74,6 +74,13 @@ withZig = false; withWasi = true; }; + # Enough to build the integration tests' wasm contracts, nothing else. + release-test-rust = import ./support/rust.nix { + inherit pkgs deps system; + withZig = false; + withWasi = true; + withCross = false; + }; custom-rust-builder = import ./support/compile-rust.nix { inherit pkgs system deps; zig = import ./support/zig.nix { inherit pkgs deps system; }; @@ -752,6 +759,16 @@ packages-0 ++ packages-rust ++ packages-debug-test ++ [ pre-commit-check.config.package ]; shellHook = shell-hook-base; }; + # Integration tests against prebuilt release tarballs: no build or fuzz tooling. + devShells.release-tests = pkgs.mkShell { + packages = packages-0 ++ [ + release-test-rust + pkgs.python312Packages.jsonnet + pkgs.python312Packages.aiohttp + pkgs.wabt + ]; + shellHook = shell-hook-base; + }; devShells.full = pkgs.mkShell { packages = packages-0 diff --git a/support/ci/pipelines/release.py b/support/ci/pipelines/release.py index 839d5148..bca76acf 100644 --- a/support/ci/pipelines/release.py +++ b/support/ci/pipelines/release.py @@ -160,7 +160,7 @@ def _cell_amd64_linux() -> dict: {'type': 'group-start', 'name': 'smoke test modules startup'}, _smoke_step('timeout'), {'type': 'group-end'}, - *stable_test_steps('.?submodules=1#mock-tests'), + *stable_test_steps('.?submodules=1#release-tests'), ], runs_on='ubuntu-latest', ) diff --git a/support/rust.nix b/support/rust.nix index d3bd212f..6ad6c3b9 100644 --- a/support/rust.nix +++ b/support/rust.nix @@ -5,6 +5,7 @@ withLinters ? false, withZig ? true, withWasi ? false, + withCross ? true, ... }@args: let @@ -44,12 +45,17 @@ let (simpleComponent manifest.pkg.cargo.target.${systemAsRust}) (simpleComponent manifest.pkg.rustc.target.${systemAsRust}) (simpleComponent manifest.pkg.rust-std.target.${systemAsRust}) - - # cross compilation - (simpleComponent manifest.pkg.rust-std.target.x86_64-unknown-linux-musl) - (simpleComponent manifest.pkg.rust-std.target.aarch64-unknown-linux-musl) - (simpleComponent manifest.pkg.rust-std.target.aarch64-apple-darwin) ] + ++ ( + if !withCross then + [ ] + else + [ + (simpleComponent manifest.pkg.rust-std.target.x86_64-unknown-linux-musl) + (simpleComponent manifest.pkg.rust-std.target.aarch64-unknown-linux-musl) + (simpleComponent manifest.pkg.rust-std.target.aarch64-apple-darwin) + ] + ) ++ ( if !withLinters then [ ] @@ -88,9 +94,8 @@ pkgs.stdenvNoCC.mkDerivation rec { buildInputs = [ pkgs.zlib pkgs.bash - - zig ] + ++ pkgs.lib.optionals withZig [ zig ] ++ pkgs.lib.optionals pkgs.stdenv.hostPlatform.isLinux [ pkgs.glibc pkgs.gcc.cc.lib @@ -140,18 +145,18 @@ pkgs.stdenvNoCC.mkDerivation rec { installPhase = '' mkdir -p $out - for i in $(find . -type d -maxdepth 2 -mindepth 1) ; + # Each tarball is `//...`; only the components belong in $out. + for i in $(find . -type d -maxdepth 2 -mindepth 2) ; do cp -r "$i/." $out/. done ls -l "$out" - - cp ${rust-objcopy} $out/bin/rust-objcopy '' + ( if withZig then '' + cp ${rust-objcopy} $out/bin/rust-objcopy wrapProgram $out/bin/cargo \ --set CC_x86_64_unknown_linux_musl zig-cc-amd64-linux \ --set CC_x86_64_unknown_linux_gnu zig-cc-amd64-linux-gnu \