From 65ee680871c3211894c0023680bd611610b03530 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 26 Oct 2026 14:42:19 -0700 Subject: [PATCH 01/28] glaeda-disk: a reserved fleet mini gets floor-pressure eviction only config/fleet-reservations.json is glaeda's copy of leo-workspace's fleet-registry reservations (a test compares the two where the registry is checked out); mini-setup installs it. On a host an active entry (or the host-local reservation.json) names, glaeda-disk runs no idle sweep and a pressure pass acts only below the runner floor, back to floor + target_margin_gib. glaeda-fleet-cas-prune leaves a reserved host above its floor alone. An unreadable list fails closed. Entries lapse by themselves. Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- config/fleet-reservations.json | 24 ++++++ scripts/glaeda-disk | 71 ++++++++++++++++++ scripts/glaeda-fleet-cas-prune | 24 ++++++ scripts/glaeda-mini-setup | 10 ++- scripts/test-glaeda-disk.py | 100 +++++++++++++++++++++++++ scripts/test-glaeda-fleet-cas-prune.py | 20 ++++- scripts/test-glaeda-mini-setup.py | 2 +- 7 files changed, 247 insertions(+), 4 deletions(-) create mode 100644 config/fleet-reservations.json diff --git a/config/fleet-reservations.json b/config/fleet-reservations.json new file mode 100644 index 000000000..8ad1c4b32 --- /dev/null +++ b/config/fleet-reservations.json @@ -0,0 +1,24 @@ +{ + "schema": "glaeda-fleet-reservations/v1", + "note": "A copy of the `reservations` list (id, owner, hosts, from, until) in leo-workspace machines/fleet-registry.json (tools/fleet_reservations.py reads the same entries), with each reserved host's registry aliases. glaeda-disk and glaeda-fleet-cas-prune read it from ~/.local/share/glaeda/fleet-reservations.json: on a host an active entry names, only floor-pressure eviction runs, at the pre-2026-10-26 ages. An entry lapses by itself after `until`. Update it with the registry: scripts/test-glaeda-disk.py compares the two where the registry is checked out.", + "reservations": [ + { + "id": "lawrence-2026-10-25", + "owner": "lawrencecchen", + "hosts": [ + "cmux16", + "cmux17", + "cmux18", + "cmux19", + "cmux20", + "cmux21", + "cmux22", + "cmux23", + "cmux24" + ], + "from": "2026-10-25T22:00:00Z", + "until": "2026-10-28T23:59:59Z" + } + ], + "aliases": {} +} diff --git a/scripts/glaeda-disk b/scripts/glaeda-disk index 12b9e15e1..2c8a5fb09 100755 --- a/scripts/glaeda-disk +++ b/scripts/glaeda-disk @@ -5212,6 +5212,64 @@ def runner_floor_gib(home: Path = HOME) -> float: return floor +# A fleet reservation (leo-workspace machines/fleet-registry.json `reservations`, shipped as +# config/fleet-reservations.json and installed here by glaeda-mini-setup) hands a host to one person for a while. On a +# reserved fleet mini only floor-pressure eviction runs: no idle sweep, and a pressure run acts only below the runner +# floor and works only to the floor plus target_margin_gib, so state its holder keeps idle is not aged out. The +# host-local fleet reservation (FLEET_ROOT/reservation.json, glaeda-reservation/v1, the one the runner hook reads) +# counts too. Fails closed: an unreadable or missing list on a fleet mini reads as reserved. +FLEET_RESERVATIONS = HOME / ".local/share/glaeda/fleet-reservations.json" +UPDATE_CONFIG = HOME / ".config/glaeda/update.json" +HOST_RESERVATION_LOCAL = FLEET_ROOT / "reservation.json" + + +def reservation_time(value: Any) -> float: + """An ISO 8601 time (Z or an offset; a naive one is UTC) as Unix seconds; ValueError otherwise.""" + if not isinstance(value, str) or not value: + raise ValueError(f"not an ISO 8601 time: {value!r}") + text = value.strip() + text = text[:-1] + "+00:00" if text[-1:] in "Zz" else text + import datetime as _dt + when = _dt.datetime.fromisoformat(text) + return (when if when.tzinfo else when.replace(tzinfo=_dt.timezone.utc)).timestamp() + + +def host_names() -> set[str]: + """This host's names as the registry spells them: glaeda-update's configured host, and the machine name without + .local, lowercased.""" + names = {platform.node().split(".")[0].lower()} + with contextlib.suppress(OSError, ValueError, AttributeError): + host = json.loads(UPDATE_CONFIG.read_text()).get("host") + if isinstance(host, str) and host: + names.add(host.lower()) + return {n for n in names if n} + + +def host_reservation(now: float | None = None, path: Path | None = None, names: set[str] | None = None, + local: Path | None = None) -> str: + """Why this host is reserved now ("" when it is not): an active fleet reservation naming it (or an alias), an + active host-local reservation, or a reservation list that cannot be read.""" + now = time.time() if now is None else now + names = host_names() if names is None else {n.lower() for n in names} + try: + doc = json.loads((FLEET_RESERVATIONS if path is None else path).read_text()) + entries, aliases = doc["reservations"], doc.get("aliases") or {} + if not isinstance(entries, list) or not isinstance(aliases, dict): + raise ValueError("malformed") + for e in entries: + hosts = {str(h).lower() for h in e["hosts"]} + hosts |= {str(a).lower() for h in e["hosts"] for a in aliases.get(h, [])} + if names & hosts and reservation_time(e["from"]) <= now <= reservation_time(e["until"]): + return f"reserved for {e.get('owner', '?')} until {e['until']} ({e.get('id', '?')})" + except (OSError, ValueError, KeyError, TypeError) as error: + return f"fleet reservations unreadable ({type(error).__name__}); kept as reserved" + with contextlib.suppress(OSError, ValueError, TypeError, AttributeError): + held = json.loads((HOST_RESERVATION_LOCAL if local is None else local).read_text()) + if held.get("schema") == "glaeda-reservation/v1" and int(held["since"]) <= now < int(held["until"]): + return f"host reservation by {held.get('owner', '?')} until {held['until']}" + return "" + + def filesystems(fams: list[Family], low: str, target: str, tmpfs_low: str = "75%", tmpfs_target: str = "90%", floor_gib: float = 0.0) -> dict[int, Fs]: out: dict[int, Fs] = {} @@ -7745,8 +7803,21 @@ def _main_impl(argv: list[str] | None = None) -> int: # home traversal. print("runner cache profile: no configured cache roots") return 0 + reserved = host_reservation() if role == "fleet-mini" and DARWIN and a.apply and not RUNNER_CACHE_PROFILE else "" try: floor_gib = max(runner_floor_gib(), a.floor_gib if math.isfinite(a.floor_gib) else 0.0) + if reserved: + RUN["reserved"] = reserved + a.idle = False # no idle sweep on a reserved host + if not a.pressure or floor_gib <= 0: + print(f"{reserved}: only floor-pressure eviction runs here" + f"{'' if a.pressure else ' (--pressure)'}{'' if floor_gib > 0 else ', and it has no runner floor'}; " + "nothing deleted") + RUN["skipped"] = "reserved host: floor-pressure eviction only" + return 0 + # pressure below the floor only, and only back to the floor plus the margin + a.low, a.target = f"{floor_gib:g}", f"{floor_gib + POLICY['target_margin_gib']:g}" + print(f"{reserved}: only floor-pressure eviction runs here") fss = filesystems(fams or [Family("home", HOME, False, "")], a.low, a.target, a.tmpfs_low, a.tmpfs_target, floor_gib) space(a.emergency, GIB) diff --git a/scripts/glaeda-fleet-cas-prune b/scripts/glaeda-fleet-cas-prune index 35b921736..ee7e55161 100755 --- a/scripts/glaeda-fleet-cas-prune +++ b/scripts/glaeda-fleet-cas-prune @@ -38,6 +38,8 @@ from __future__ import annotations import argparse import errno +import importlib.machinery +import importlib.util import fcntl import json import os @@ -60,6 +62,24 @@ GC_EVERY = 86400 # seconds between fleet-store gc runs GC_REPORT = re.compile(r"kept entries naming no stored object: (\d+)") +def reserved_above_floor() -> str: + """Why this host is left alone: a fleet reservation holds it (glaeda-disk's host_reservation, read from the + glaeda-disk installed beside this script) and free disk is at or above its runner floor. "" when not.""" + try: + loader = importlib.machinery.SourceFileLoader("glaeda_disk_for_prune", os.fspath(Path(__file__).with_name("glaeda-disk"))) + spec = importlib.util.spec_from_loader(loader.name, loader) + gd = importlib.util.module_from_spec(spec) + loader.exec_module(gd) + why = gd.host_reservation() + floor = gd.runner_floor_gib() + except Exception as error: # noqa: BLE001 (fail closed: no glaeda-disk to ask, so treat the host as reserved) + return f"cannot read fleet reservations ({type(error).__name__}); kept as reserved" + if not why: + return "" + free = shutil.disk_usage(ROOT if ROOT.is_dir() else Path.home()).free / GIB + return "" if floor > 0 and free < floor else f"{why}; {free:.0f} GiB free, floor {floor:g}" + + def busy(commands: list[str]) -> str: for c in commands: if BUSY.search(c): @@ -206,6 +226,10 @@ def main(argv: list[str] | None = None) -> int: return 0 writer = is_writer(ROOT) rec["role"] = "writer" if writer else "reader" + if a.apply and (held := reserved_above_floor()): # a reserved host's caches are its holder's until the floor + rec["result"] = f"deferred: {held}" + print(json.dumps(rec)) + return 0 commands = running_commands() if commands is None: rec["result"] = "deferred: ps failed, cannot rule out a running build" diff --git a/scripts/glaeda-mini-setup b/scripts/glaeda-mini-setup index 2d472bc86..dff6d5d09 100755 --- a/scripts/glaeda-mini-setup +++ b/scripts/glaeda-mini-setup @@ -194,6 +194,7 @@ DAEMON_UNITS = frozenset({"glaeda-gh.service"}) SKILL_SOURCE = REPO / "skills/github-ci-wait/SKILL.md" DAEMON_POLICY_SOURCE = REPO / "config/macos-daemons.json" # desired state of macOS system daemons CLAUDE_ROUTES_SOURCE = REPO / "config/claude-routes.json" # the fleet routes glaeda-mini-health keeps Claude on +FLEET_RESERVATIONS_SOURCE = REPO / "config/fleet-reservations.json" # who holds which fleet host; glaeda-disk reads it # glaeda-update runs this command from a release; it sets this so setup never unloads the job running it UPDATE_ENV = "GLAEDA_UPDATE_RUNNING" OTA_RINGS = ("canary", "stable") @@ -987,6 +988,11 @@ def skill_path(ctx: Context) -> Path: return ctx.home / ".claude/skills/github-ci-wait/SKILL.md" +def fleet_reservations_path(ctx: Context) -> Path: + """config/fleet-reservations.json, where glaeda-disk and glaeda-fleet-cas-prune read it (FLEET_RESERVATIONS).""" + return ctx.home / ".local/share/glaeda/fleet-reservations.json" + + def daemon_policy_path(ctx: Context) -> Path: """config/macos-daemons.json, where glaeda-mini-health and glaeda-daemon-heal read it.""" return ctx.home / ".local/share/glaeda/macos-daemons.json" @@ -1286,7 +1292,8 @@ def plan_install(ctx: Context) -> list[dict[str, Any]]: def plan_hygiene_tools(ctx: Context) -> list[dict[str, Any]]: """The macOS daemon policy and glaeda-worktree-reclaim: the parts of a hygiene install beyond the scripts.""" actions: list[dict[str, Any]] = [] - for dest, source in ((daemon_policy_path(ctx), DAEMON_POLICY_SOURCE), (claude_routes_path(ctx), CLAUDE_ROUTES_SOURCE)): + for dest, source in ((daemon_policy_path(ctx), DAEMON_POLICY_SOURCE), (claude_routes_path(ctx), CLAUDE_ROUTES_SOURCE), + (fleet_reservations_path(ctx), FLEET_RESERVATIONS_SOURCE)): act = file_action(ctx, "tool", dest, source.read_bytes(), 0o644) act["source"] = os.fspath(source) actions.append(act) @@ -1483,6 +1490,7 @@ def plan_uninstall(ctx: Context) -> list[dict[str, Any]]: actions.append({"kind": "tool", "path": os.fspath(skill_path(ctx))}) actions.append({"kind": "tool", "path": os.fspath(daemon_policy_path(ctx))}) actions.append({"kind": "tool", "path": os.fspath(claude_routes_path(ctx))}) + actions.append({"kind": "tool", "path": os.fspath(fleet_reservations_path(ctx))}) for marker in scan_exclusion_markers(ctx): actions.append({"kind": "marker", "path": os.fspath(marker)}) for act in actions: diff --git a/scripts/test-glaeda-disk.py b/scripts/test-glaeda-disk.py index 46fb202ab..fdc50559c 100644 --- a/scripts/test-glaeda-disk.py +++ b/scripts/test-glaeda-disk.py @@ -39,6 +39,13 @@ gd = importlib.util.module_from_spec(spec) sys.modules["glaeda_disk"] = gd loader.exec_module(gd) +# Every test runs on an unreserved host unless it says otherwise (HostReservationTest): an empty reservation list, so a +# test's host never reads the real ~/.local/share/glaeda/fleet-reservations.json, or fails closed without one. +_NO_RESERVATIONS = tempfile.NamedTemporaryFile("w", suffix="-fleet-reservations.json", delete=False) +_NO_RESERVATIONS.write('{"schema": "glaeda-fleet-reservations/v1", "reservations": [], "aliases": {}}') +_NO_RESERVATIONS.close() +gd.FLEET_RESERVATIONS = Path(_NO_RESERVATIONS.name) +gd.HOST_RESERVATION_LOCAL = Path(_NO_RESERVATIONS.name + ".none") # An --apply in a test must never reap the real $TMPDIR of the host running the suite: without a tmp of # the test's own, the TMPDIR reaper does nothing. _reap_tmpdir_test_residue = gd.reap_tmpdir_test_residue @@ -6190,5 +6197,98 @@ def fss(*args: object) -> dict: gd.main(["--pressure", "--need", "50", "--no-snapshot"]) self.assertEqual(seen, [("54", "70", 30.0), ("55", "71", 50.0)]) +class HostReservationTest(unittest.TestCase): + """A reserved fleet mini gets floor-pressure eviction only, at the pre-2026-10-26 ages (config/fleet-reservations.json).""" + + def setUp(self) -> None: + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.list = self.root / "fleet-reservations.json" + self.local = self.root / "reservation.json" + self.list.write_text(json.dumps({"schema": "glaeda-fleet-reservations/v1", "aliases": {"cmux30": ["cmux-thirty"]}, + "reservations": [{"id": "r1", "owner": "someone", "hosts": ["cmux30"], + "from": "2026-10-25T22:00:00Z", + "until": "2026-10-28T23:59:59Z"}]})) + self.during = gd.reservation_time("2026-10-27T00:00:00Z") + + def reserved(self, names, now=None) -> str: + return gd.host_reservation(self.during if now is None else now, self.list, set(names), self.local) + + def test_an_active_entry_or_alias_reserves_and_it_lapses_by_itself(self) -> None: + self.assertIn("reserved for someone until 2026-10-28T23:59:59Z (r1)", self.reserved({"cmux30"})) + self.assertIn("reserved for someone", self.reserved({"CMUX-Thirty"})) + self.assertEqual(self.reserved({"cmux31"}), "") + self.assertEqual(self.reserved({"cmux30"}, gd.reservation_time("2026-10-29T00:00:00Z")), "", "lapsed") + self.assertEqual(self.reserved({"cmux30"}, gd.reservation_time("2026-10-25T21:59:59Z")), "", "not yet") + + def test_an_unreadable_list_fails_closed(self) -> None: + self.list.write_text("{nope") + self.assertIn("unreadable", self.reserved({"cmux31"})) + self.list.unlink() + self.assertIn("unreadable", self.reserved({"cmux31"})) + + def test_the_host_local_reservation_counts(self) -> None: + now = int(self.during) + self.local.write_text(json.dumps({"schema": "glaeda-reservation/v1", "owner": "fleet", "since": now - 60, + "until": now + 60})) + self.assertIn("host reservation by fleet", self.reserved({"cmux31"})) + self.local.write_text(json.dumps({"schema": "glaeda-reservation/v1", "owner": "fleet", "since": now - 60, + "until": now - 1})) + self.assertEqual(self.reserved({"cmux31"}), "") + + def test_the_shipped_list_parses_and_matches_the_registry_where_it_is_checked_out(self) -> None: + shipped = json.loads((ROOT / "config/fleet-reservations.json").read_text()) + for e in shipped["reservations"]: + gd.reservation_time(e["from"]), gd.reservation_time(e["until"]) + self.assertTrue(e["hosts"] and e["owner"] and e["id"]) + registry = Path(os.environ.get("GLAEDA_FLEET_REGISTRY") + or Path.home() / "Projects/leo-workspace/machines/fleet-registry.json") + if not registry.is_file(): + self.skipTest("leo-workspace's fleet registry is not checked out here") + want = [{k: e[k] for k in ("id", "owner", "hosts", "from", "until") if k in e} + for e in json.loads(registry.read_text()).get("reservations", [])] + self.assertEqual(shipped["reservations"], want, "config/fleet-reservations.json is behind the registry") + + def run_disk(self, *args: str, reserved: str, floor: float, free_gib: float): + """glaeda-disk's main on a fake fleet mini; returns (exit code, output, the families' low/target it used).""" + seen, run = {}, {} + + def filesystems(fams, low, target, *rest): + seen.update(low=low, target=target) + run.update(gd.RUN) + raise SystemExit(0) # stop before any survey: the gate is what is under test + with mock.patch.object(gd, "host_reservation", return_value=reserved), \ + mock.patch.object(gd, "runner_floor_gib", return_value=floor), \ + mock.patch.object(gd, "DARWIN", True), mock.patch.object(gd, "filesystems", side_effect=filesystems), \ + mock.patch.object(gd, "disk_off", return_value=""), mock.patch.dict(gd.RUN, clear=True), \ + contextlib.redirect_stdout(io.StringIO()) as out: + try: + code = gd._main_impl(["--apply", "--top", "0", "--no-snapshot", *args]) + except SystemExit as stop: + code = stop.code + return code, out.getvalue(), seen, run + + def test_a_reserved_host_runs_only_floor_pressure_to_the_floor_plus_the_margin(self) -> None: + why = "reserved for someone until 2026-10-28T23:59:59Z (r1)" + for args in (("--idle",), ()): + with self.subTest(args=args): + code, out, seen, run = self.run_disk(*args, reserved=why, floor=30.0, free_gib=50) + self.assertEqual(code, 0) + self.assertIn("only floor-pressure eviction runs here", out) + self.assertIn("nothing deleted", out) + self.assertEqual(seen, {}, "never reached a survey") + code, out, seen, run = self.run_disk("--pressure", "--idle", reserved=why, floor=0.0, free_gib=50) + self.assertIn("no runner floor", out) + self.assertEqual(seen, {}) + code, out, seen, run = self.run_disk("--pressure", "--idle", reserved=why, floor=30.0, free_gib=50) + margin = gd.POLICY["target_margin_gib"] + self.assertEqual(seen, {"low": "30", "target": f"{30 + margin:g}"}) + self.assertEqual(run["reserved"], why) + # an unreserved host keeps its policy + code, out, seen, run = self.run_disk("--pressure", reserved="", floor=30.0, free_gib=50) + self.assertEqual(seen, {"low": gd.POLICY["low"], "target": gd.POLICY["target"]}) + + if __name__ == "__main__": unittest.main() diff --git a/scripts/test-glaeda-fleet-cas-prune.py b/scripts/test-glaeda-fleet-cas-prune.py index 75f1790ad..ae0886fe4 100644 --- a/scripts/test-glaeda-fleet-cas-prune.py +++ b/scripts/test-glaeda-fleet-cas-prune.py @@ -14,6 +14,7 @@ import tempfile import time import unittest +from unittest import mock from pathlib import Path ROOT = Path(__file__).resolve().parents[1] @@ -32,12 +33,13 @@ def setUp(self) -> None: self.root.mkdir() self.lock = Path(self.tmp.name) / "host.lock" self.lock.write_text("") - self.saved = (fp.ROOT, fp.HOST_LOCK, fp.running_commands) + self.saved = (fp.ROOT, fp.HOST_LOCK, fp.running_commands, fp.reserved_above_floor) fp.ROOT, fp.HOST_LOCK = self.root, self.lock fp.running_commands = lambda: ["/usr/libexec/something", "zsh"] + fp.reserved_above_floor = lambda: "" def tearDown(self) -> None: - fp.ROOT, fp.HOST_LOCK, fp.running_commands = self.saved + fp.ROOT, fp.HOST_LOCK, fp.running_commands, fp.reserved_above_floor = self.saved self.tmp.cleanup() def env(self, sign_key: str = "") -> None: @@ -190,5 +192,19 @@ def test_defers_while_a_build_runs_or_holds_the_lock(self) -> None: self.assertTrue(all(p.exists() for p in paths)) + def test_a_reserved_host_above_its_floor_is_left_alone(self) -> None: + self.env() + paths = self.node(10) + fp.reserved_above_floor = lambda: "reserved for someone until 2026-10-28T23:59:59Z (r1); 80 GiB free, floor 30" + rec = self.run_main("--apply", "--node-gib", "0") + self.assertTrue(rec["result"].startswith("deferred: reserved for someone"), rec) + self.assertTrue(all(p.exists() for p in paths)) + self.assertIn("would remove", self.run_main("--node-gib", "0")["node_store"], "a plan still reports") + + def test_reserved_above_floor_reads_glaeda_disk_and_fails_closed(self) -> None: + real = self.saved[3] + with mock.patch.object(fp.importlib.machinery, "SourceFileLoader", side_effect=OSError("gone")): + self.assertIn("kept as reserved", real()) + if __name__ == "__main__": unittest.main() diff --git a/scripts/test-glaeda-mini-setup.py b/scripts/test-glaeda-mini-setup.py index 2686c5f99..da1f4c0eb 100644 --- a/scripts/test-glaeda-mini-setup.py +++ b/scripts/test-glaeda-mini-setup.py @@ -713,7 +713,7 @@ def test_hygiene_only_installs_tools_and_agents_on_any_mac(self) -> None: self.assertTrue(os.access(self.home / ".local/bin" / name, os.X_OK), name) for label in ("disk-pressure", "disk-dedupe", "worktree-reclaim"): self.assertTrue((self.home / f"Library/LaunchAgents/com.teamleaderleo.glaeda.{label}.plist").is_file()) - for name in ("macos-daemons.json", "claude-routes.json"): # what glaeda-mini-health reads off the repo + for name in ("macos-daemons.json", "claude-routes.json", "fleet-reservations.json"): # read off the repo self.assertEqual((self.home / ".local/share/glaeda" / name).read_bytes(), (ROOT / "config" / name).read_bytes(), name) # the build-host profile still blocks on the same machine From 942e49f6e61d429834ff06959212dd2cbf51e449 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 26 Oct 2026 14:52:15 -0700 Subject: [PATCH 02/28] glaeda-disk: judge the reservation before any family is built Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- scripts/glaeda-disk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/glaeda-disk b/scripts/glaeda-disk index 2c8a5fb09..c3a261c00 100755 --- a/scripts/glaeda-disk +++ b/scripts/glaeda-disk @@ -7762,6 +7762,8 @@ def _main_impl(argv: list[str] | None = None) -> int: a.low = f"{a.need + POLICY['low_margin_gib']:g}" a.target = f"{a.need + POLICY['target_margin_gib']:g}" + # before any family is built: a reserved host's families keep its holder's ages (keep_legacy_ages, once 12 h lands) + reserved = host_reservation() if role == "fleet-mini" and DARWIN and a.apply and not a.runner_caches else "" global RUNNER_CACHE_PROFILE RUNNER_CACHE_PROFILE = bool(a.runner_caches) fams = runner_cache_families() if RUNNER_CACHE_PROFILE else default_families(role) @@ -7803,7 +7805,6 @@ def _main_impl(argv: list[str] | None = None) -> int: # home traversal. print("runner cache profile: no configured cache roots") return 0 - reserved = host_reservation() if role == "fleet-mini" and DARWIN and a.apply and not RUNNER_CACHE_PROFILE else "" try: floor_gib = max(runner_floor_gib(), a.floor_gib if math.isfinite(a.floor_gib) else 0.0) if reserved: From 5cf61c2e4d84d53b8479ecf1ec10b2b6123ecd03 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 26 Oct 2026 14:06:58 -0700 Subject: [PATCH 03/28] glaeda-disk: every disposable age is 12 h; runner checkouts unused 12 h go above the floor Leo, 2026-10-26: "When have we ever had something useful greater than 12 hours?" DerivedData, CoreSimulator runtimes, package caches, runner work dirs and build dirs now go once idle DEFAULT_AGE_HOURS (12 h). The per-host cmux-next build root, the newest warm compile cache per builder, is exempt by role (only under the floor). The git-mirror window stays 168 h: past it a checkout's unpushed commits are not checked. Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- scripts/glaeda-disk | 111 +++++++++++++++++++++--------------- scripts/test-glaeda-disk.py | 65 +++++++++++++++++---- 2 files changed, 119 insertions(+), 57 deletions(-) diff --git a/scripts/glaeda-disk b/scripts/glaeda-disk index c3a261c00..bb1d13a82 100755 --- a/scripts/glaeda-disk +++ b/scripts/glaeda-disk @@ -71,7 +71,7 @@ Runner.Worker vetoes still hold (docs/CMUX_NEXT_CI_RETENTION.md). Only a pressur report shows its last measurement, and its run line records how far over the cap it is. The per-host build root (cmux_next_ci_host_roots, default dev-daily-release/host: one tree a mini's DEV compiles share under flock on its lock, so their compilation cache hits) is never the cap's: it goes whole, after -everything else, only once idle cmux_next_ci_host_idle_hours (48) or while free is still under the runner floor, +everything else, only while free is still under the runner floor (exempt from age by role: the newest warm cache), and never while its lock is held. Per-instance dev-daily-release states, now its fallback only, go first. Each enrolled GitHub runner's checkout (~/actions-runner*/_work//) has two families. @@ -79,9 +79,9 @@ runner-checkout-build is SwiftPM's Packages/*/.build and Packages/*/*/.build plu directories (cmux-tui/target), plus DerivedData a workflow keeps in $RUNNER_WORKSPACE/.glaeda-state (_work//.glaeda-state/*derived-data*); it goes only under pressure, only while that runner has no Runner.Worker, an hour idle, least recently modified first. runner-git-garbage is git's tmp_pack_*/tmp_obj_* temp files an -interrupted fetch left; one an hour old goes whenever nothing holds it open. The checkout itself stays, -except as the last resort of a pressure run still under the runner gate's disk floor after everything else -(runner-idle-checkout): then the whole checkout of a runner with no Runner.Worker, unused for 10 minutes and +interrupted fetch left; one an hour old goes whenever nothing holds it open. The checkout itself stays +until no job used it for 12 hours (any pressure or sweep pass then removes it), except as the last resort of a +pressure run still under the runner gate's disk floor after everything else (runner-idle-checkout): then the whole checkout of a runner with no Runner.Worker, unused for 10 minutes and held open by nothing, goes, least recently used first, until free space reaches the floor plus target_margin_gib. The runner's next job clones it again. Before that, on every pressure or sweep pass of a fleet mini, runner-checkout-repack folds a checkout's packs with `git repack -a -d -k -l` (keeps every object) once @@ -101,7 +101,7 @@ it or its linked work trees. Only a home with enrolled runners. A slow gc runs o The runner-account profile is intentionally narrower than the normal user profile: it covers only DerivedData, cmux reload/GhosttyKit caches, and CUA recordings/frame -caches, with a seven-day minimum idle age. Git index refreshes are ignored for that +caches, with a DEFAULT_AGE_HOURS (12 h) minimum idle age. Git index refreshes are ignored for that age clock; process/open-file/command/socket/git-loss checks still veto deletion. The idle sweep (--idle) reclaims whatever the free space. A busy host sweeps too: deletes skip only what a @@ -551,13 +551,19 @@ def artifact_referenced(path: Path, references: frozenset[str]) -> bool: FLEET_CI = Path("/Users/Shared/cmux-build-fleet/ci") SEED_SOURCE = "seed-source.json" SEED_KEEP = 2 # seed_derived_data.LOCAL_KEEP_LOW_DISK: newest seeds of the current prefix each root keeps -KEPT_BUILD_IDLE_HOURS = 24.0 +# One age for everything disposable (Leo, 2026-10-26): "When have we ever had something useful greater than 12 hours? +# I mean, if something stalls, like sure, but why would it stall? If it stalls, it's gonna have to get a rebase +# anyway." DerivedData, CoreSimulator, package caches, runner work dirs and build dirs go once idle this long. Only +# the newest warm compile cache per builder is exempt, by role and not by age (the per-host cmux-next build root, +# cmux_next_ci_host_idle_hours; each root's newest SEED_KEEP seeds); older warm caches go at this age too. +DEFAULT_AGE_HOURS = 12.0 +KEPT_BUILD_IDLE_HOURS = DEFAULT_AGE_HOURS PARKED_IDLE_HOURS = float("inf") # parked builds age by host-job reuse distance, not wall clock # A missing job log cannot establish a reuse distance. An idle sweep can still make -# bounded progress after the same seven-day window used for mirror checkouts; the +# bounded progress after DEFAULT_AGE_HOURS, the window used for mirror checkouts; the # sweep's host-busy gate plus the normal process, socket, lease and recheck guards # remain in force before any deletion. -PARKED_NO_LOG_IDLE_HOURS = 168.0 +PARKED_NO_LOG_IDLE_HOURS = DEFAULT_AGE_HOURS def fleet_stores(ci: Path | None = None) -> list[Path]: @@ -576,7 +582,7 @@ FLEET_FAMILY_IDS = ("cmux-pr-builds", "cmux-seeds", "cmux-kept-build", "ci-deriv # are deliberately outside this allowlist. CMUX_NEXT_CI_FAMILY = "cmux-next-ci-generated" CMUX_NEXT_CI_ROOT = "cmux-next-ci" -CMUX_NEXT_CI_MIN_IDLE_HOURS = 72.0 +CMUX_NEXT_CI_MIN_IDLE_HOURS = DEFAULT_AGE_HOURS CMUX_NEXT_CI_DRAIN = False # Bytes the whole cmux-next-ci cache is over its LRU cap (cmux_next_ci_cap_gib) in this pressure run, 0 within # it. While over, the 72 h window gives way, least recently used first, until the cache is back under the cap @@ -613,6 +619,7 @@ CMUX_NEXT_DRAIN_HOOK_SHA256 = frozenset({ "d5e73be990d67d9499ff352d01fda2e1392a6fd257a57770b23008b3bf55fb59", "0c2c8880e3b17af8324f8304121c4159bee8ade5c31496b9af713586cb90bcfc", "fca8abb34684680ad0344a7a82b0ee8b2b476f6490407de27eb410904f5fac91", + "6bdc89db3e25e5fa40321e5d3066d7d5cf4ba7354c6915687cd76f118d7dfec5", }) CMUX_NEXT_CI_PRESERVED_NAMES = frozenset({ "SourcePackages", "ModuleCache.noindex", "SDKStatCaches.noindex", @@ -692,7 +699,7 @@ def ci_ios_family(root: Path | None = None) -> Family: # (2026-10-21). Only these names one level below the root, so src/ stays the tmp family's and git's to judge. CANONICAL_ADMISSION = "cmux-ci-admission" CANONICAL_ADMISSION_NAMES = ("derived-data-", "compile-admission-cas") -CANONICAL_ADMISSION_IDLE_HOURS = 24.0 +CANONICAL_ADMISSION_IDLE_HOURS = DEFAULT_AGE_HOURS CANONICAL_CAPACITY = FLEET_ROOT / "capacity" @@ -944,7 +951,7 @@ def item_window(fam: Family, path: Path, idle_hours: float, archived: frozenset[ if fam.root.stat().st_dev in FLOOR_HOLD: return 0.0 window = float(POLICY["cmux_next_ci_host_idle_hours"]) - return max(window, fam.min_idle_hours) if RUNNER_CACHE_PROFILE else window # that profile's 7-day minimum + return max(window, fam.min_idle_hours) if RUNNER_CACHE_PROFILE else window # that profile's minimum age if fam.id == CMUX_NEXT_CI_FAMILY and (CMUX_NEXT_CI_DRAIN or cmux_next_ci_over()): return 0.0 if fam.id == SPM_SCRATCH: @@ -1005,7 +1012,7 @@ def cmux_next_cap_only(fam: Family, path: Path, idle: float, idle_hours: float) # which bounds stale disposable output without touching a live job's state. IDLE_SWEEP_HOURS = KEPT_BUILD_IDLE_HOURS IDLE_PR_BUILD_HOURS = float("inf") -IDLE_SEED_HOURS = 12.0 +IDLE_SEED_HOURS = DEFAULT_AGE_HOURS IDLE_SWEEP_EVERY_S = 3600 # the 15-minute agent sweeps at most hourly: an apply re-measures every candidate IDLE_STAMP = HOME / ".cache/glaeda-disk/idle-sweep.last" FLEET_DIR = FLEET_CI.parent @@ -1127,12 +1134,12 @@ POLICY_DEFAULTS: dict[str, Any] = { "pressure_jobs": 80, "min_jobs": 6, "sweep_jobs": 190, - "soft_idle_hours": 24.0, + "soft_idle_hours": DEFAULT_AGE_HOURS, # Build markers are aged by observed host activity, not a fixed wall-clock # guess. The cap is only a recovery bound when the job log is unavailable. "stale_marker_min_age_hours": 0.5, "stale_marker_jobs": 3, - "stale_marker_max_hours": 48.0, + "stale_marker_max_hours": DEFAULT_AGE_HOURS, # A busy host cleans too: deletes skip only what a job uses (open, named by a command, its root's # current seed or checkout), never the whole pass. true restores the old defer-while-CI-runs rule. "defer_during_ci": False, @@ -1148,7 +1155,9 @@ POLICY_DEFAULTS: dict[str, Any] = { # (cmux_next_ci_host_roots): never for the size cap, only once idle cmux_next_ci_host_idle_hours, or while free is # still under the runner floor after everything else, and never while its lock is held. [] turns this off. "cmux_next_ci_host_roots": ["dev-daily-release/host"], - "cmux_next_ci_host_idle_hours": 48.0, + # The newest warm compile cache per builder is exempt by role, not by age (DEFAULT_AGE_HOURS): the host root goes + # only under the runner floor. A finite number here ages it again. + "cmux_next_ci_host_idle_hours": float("inf"), # --dedupe also clones identical files in capture and evidence trees (screen-capture frames, UI evidence), # with its own floor: one PNG frame is 16-200 KiB, so Xcode's 1 MiB floor saw 2.7 of the 11.2 GiB of # identical frames in cmux-mac-mini's ~/messageslab-26. "~" is the account's home; globs are allowed; a @@ -1514,7 +1523,7 @@ def idle_due(stamp: Path | None = None, every_s: float = IDLE_SWEEP_EVERY_S) -> # Xcode copies and simulator runtimes (idle sweep, macOS). An Xcode is an install, not a rebuild: it is only # reported, with the pins that name it. A runtime is a multi-GiB download; one goes only when nothing uses it. -RUNTIME_IDLE_DAYS = 14.0 # simctl's lastUsedAt: an iOS lane touches its runtime on every run +RUNTIME_IDLE_DAYS = DEFAULT_AGE_HOURS / 24 # simctl's lastUsedAt: an iOS lane touches its runtime on every run def _version_key(v: str) -> tuple[int, ...]: @@ -1694,7 +1703,7 @@ def seed_verdict(path: Path) -> str: WORKTREE_DIRS = ("worktrees", "cmux-worktrees", "glaeda-worktrees") -CHECKOUT_BUILD_IDLE_HOURS = 24.0 +CHECKOUT_BUILD_IDLE_HOURS = DEFAULT_AGE_HOURS def _checkout_git(checkout: Path, *args: str) -> list[str]: @@ -1775,7 +1784,7 @@ def checkout_build_family(projects: Path, role: str = "fleet-mini") -> Family: return Family("checkout-build", projects, True, "swift build or glaeda-apple warm rebuilds it; also counted in its checkout", checkout_builds=True, build_output=True, - min_idle_hours=min(CHECKOUT_BUILD_IDLE_HOURS, 12.0), + min_idle_hours=CHECKOUT_BUILD_IDLE_HOURS, owner="SwiftPM and glaeda-apple, per checkout", rebuild_minutes=15, platform="shared") @@ -1912,6 +1921,9 @@ def runner_checkout_families(home: Path | None = None) -> list[Family]: # used 23-32 minutes before, and the pass freed nothing). No Runner.Worker is what keeps a job's checkout. RUNNER_IDLE_CHECKOUT_FAMILY = "runner-idle-checkout" RUNNER_IDLE_CHECKOUT_HOURS = 10 / 60 +# Old runner work dirs go by age too, above the floor and in the idle sweep: a checkout no job used for +# DEFAULT_AGE_HOURS is cloned again by its runner's next job, like one the floor took. +RUNNER_STALE_CHECKOUT_HOURS = DEFAULT_AGE_HOURS def runner_checkout_last_use(runner: Path, checkout: Path) -> float: @@ -1948,17 +1960,16 @@ def idle_runner_checkout_veto(runner: Path, checkout: Path, opened: frozenset[st def evict_idle_runner_checkouts(receipt: Path, floor_gib: float, home: Path | None = None) -> int: - """Under the runner gate's disk floor only: remove idle runners' whole checkouts, least recently used - first, each after fresh process evidence, until free space is back at the floor plus target_margin_gib.""" + """Remove idle runners' whole checkouts, least recently used first, each after fresh process evidence: every + one unused for RUNNER_STALE_CHECKOUT_HOURS, and under the runner gate's disk floor also younger ones (quiet for + RUNNER_IDLE_CHECKOUT_HOURS) until free space is back at the floor plus target_margin_gib.""" home = HOME if home is None else home - if floor_gib <= 0: - return 0 try: free, total = free_bytes(home) except OSError: return 0 - if free >= (floor_gib + GATE_DISK_RESUME_GIB) * GIB: # the gate is not holding for disk: whole checkouts stay - return 0 + # above the gate's resume mark (or with no floor) only stale checkouts go + under = floor_gib > 0 and free < (floor_gib + GATE_DISK_RESUME_GIB) * GIB target = min(int((floor_gib + POLICY["target_margin_gib"]) * GIB), total // 2) # as filesystems() sets it now = time.time() found = [] @@ -1970,13 +1981,15 @@ def evict_idle_runner_checkouts(receipt: Path, floor_gib: float, home: Path | No if checkout.name != checkout.parent.name: # only actions/checkout's default _work// continue used = runner_checkout_last_use(runner, checkout) - if (now - used) / 3600 >= RUNNER_IDLE_CHECKOUT_HOURS: + idle = (now - used) / 3600 + if idle >= RUNNER_IDLE_CHECKOUT_HOURS and (under or idle >= RUNNER_STALE_CHECKOUT_HOURS): found.append((used, runner, checkout)) released = 0 for used, runner, checkout in sorted(found, key=lambda t: t[0]): + stale = (now - used) / 3600 >= RUNNER_STALE_CHECKOUT_HOURS try: - if free_bytes(home)[0] >= target: - break + if not stale and free_bytes(home)[0] >= target: + break # oldest first: every stale checkout came before this one opened, cmds = process_evidence() # a job may have started since the last deletion except OSError: break @@ -1999,12 +2012,14 @@ def evict_idle_runner_checkouts(receipt: Path, floor_gib: float, home: Path | No rec = {"at": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "host": platform.node(), "outcome": outcome, "family": RUNNER_IDLE_CHECKOUT_FAMILY, "path": os.fspath(checkout), "bytes": size, "verdict": "reclaimable" if not why else "blocked", - "reasons": [why] if why else ["under the runner disk floor", "runner has no job"], + "reasons": [why] if why else [f"unused for {RUNNER_STALE_CHECKOUT_HOURS:g} h" if stale + else "under the runner disk floor", "runner has no job"], "idle_hours": round((now - used) / 3600, 2)} with contextlib.suppress(OSError), receipt.open("a") as f: # a lost receipt line must not stop the pass f.write(json.dumps(rec) + "\n") if outcome == "reclaimed": - print(f" reclaimed {gib(size).strip():>8s} {checkout} (idle runner checkout, under the disk floor)") + why_now = f"unused for {RUNNER_STALE_CHECKOUT_HOURS:g} h" if stale else "under the disk floor" + print(f" reclaimed {gib(size).strip():>8s} {checkout} (idle runner checkout, {why_now})") return released @@ -2021,7 +2036,7 @@ TMP_TEST_RESIDUE = re.compile( r"|floating-chat|header-pr|picker-listing|swift-package-test\.XXXXXX|TemporaryDirectory|cmux-claude-hooks" r"|swift-crash-reproducer|cmux-shortcut-routing|cmux-agent-journal-test|cmux-git-active-pwd" r"|cmux-ssh-pty-replay)[-.][0-9A-Za-z]") -TMP_TEST_RESIDUE_IDLE_HOURS = 48.0 # nothing in the entry changed for two days: no test run still uses it +TMP_TEST_RESIDUE_IDLE_HOURS = DEFAULT_AGE_HOURS # nothing in the entry changed for this long: no test run uses it TMP_TEST_RESIDUE_WALK_BUDGET = 20_000 # entries walked per candidate; a bigger tree is not proven idle TMP_TEST_RESIDUE_EVIDENCE_S = 60.0 # process evidence is re-read at least this often during a pass TMP_TEST_RESIDUE_BUDGET_S = 600.0 # one pass stops here and holds the eviction lock no longer; the next continues @@ -2296,6 +2311,8 @@ HQ_CACHE_FAMILY = "hq-build-fleet-cache" # laptop, which keeps the edits. So once a unit sat idle a week, no git check keeps it. Before that, a clone # still has to lose nothing. Every git check kept them: 107 GiB idle since Sep 28-Oct 2 on eight minis read # "git-checkout (checkout below the top level)" while glaeda-disk reported 0.0 G reclaimable (2026-10-11). +# Not DEFAULT_AGE_HOURS: past this window a checkout counts as a mirror and goes without the git-loss check, so a +# commit only it holds would be lost unarchived; the disposable output inside it goes at DEFAULT_AGE_HOURS. CMUX_JOB_MIRROR_HOURS = 168.0 @@ -2365,7 +2382,7 @@ def default_families(role: str = "fleet-mini") -> list[Family]: # runs therefore cannot erase a slot used recently in the same work period. Family("cmux-job-cache", HOME / ".cache/cmux-job", True, "next cmux reload or CI job rebuilds it (cold)", depth=2, git_disposable=True, - owner="cmuxterm-hq reload-cloud scripts", rebuild_minutes=30, min_idle_hours=12.0, + owner="cmuxterm-hq reload-cloud scripts", rebuild_minutes=30, min_idle_hours=DEFAULT_AGE_HOURS, mirror_idle_hours=CMUX_JOB_MIRROR_HOURS, platform=host_cache_platform()), # Caches are disposable by definition; an idle one goes under pressure, a live one (go-build # while a Go build runs) stays recent. Any git data inside keeps it, as everywhere else. @@ -2387,17 +2404,18 @@ def default_families(role: str = "fleet-mini") -> list[Family]: # can see their age and reclaim them after the normal process/open-file gate passes. Family("cua-recordings", HOME / "Library/Application Support/cua-ssh/recordings", True, "the CUA recorder can capture it again", depth=1, files=True, - min_idle_hours=24.0, owner="CUA recorder", rebuild_minutes=2, platform="macos"), + min_idle_hours=DEFAULT_AGE_HOURS, owner="CUA recorder", rebuild_minutes=2, platform="macos"), Family("cua-cache", HOME / "Library/Caches/cmux/cmux-cua", True, "the CUA client rebuilds its frame cache", depth=1, files=True, - min_idle_hours=12.0, owner="cmux CUA", rebuild_minutes=1, platform="macos"), + min_idle_hours=DEFAULT_AGE_HOURS, owner="cmux CUA", rebuild_minutes=1, platform="macos"), # Sealed caches (cache_layout) anywhere else in the home directory: ~/.npm/_cacache, # ~/.bun/install/cache, ~/.cargo/registry, ~/go/pkg/mod, rustup toolchains. Found by content, # so a tool this list never heard of is covered once its cache has a recognized layout. - # Twelve hours at least: Linux's relatime moves a file's access time once a day, so a cache read - # every hour can show a day-old atime. The live process and command checks still protect use. + # DEFAULT_AGE_HOURS: Linux's relatime moves a file's access time once a day, so a cache read every hour can + # show a day-old atime; the live process and command checks protect a cache in use, and one that goes is + # downloaded again. Family("home-caches", HOME, True, "the tool re-downloads or rebuilds it", discover=True, - owner="tools (found by cache layout)", rebuild_minutes=5, min_idle_hours=24.0, pressure_only=role == "devbox", platform=host_cache_platform()), + owner="tools (found by cache layout)", rebuild_minutes=5, min_idle_hours=DEFAULT_AGE_HOURS, pressure_only=role == "devbox", platform=host_cache_platform()), Family("simulators", HOME / "Library/Developer/CoreSimulator", False, "use `xcrun simctl delete unavailable` (report only)", platform="macos"), Family("projects", projects, False, "checkouts (report only)", @@ -2512,26 +2530,26 @@ def runner_cache_families() -> list[Family]: roots = [ Family("xcode-derived-data", HOME / "Library/Developer/Xcode/DerivedData", True, "next xcodebuild rebuilds it", owner="Xcode, per checkout", rebuild_minutes=15, - build_output=True, min_idle_hours=168.0, platform="macos"), + build_output=True, min_idle_hours=DEFAULT_AGE_HOURS, platform="macos"), Family("cmux-job-cache", HOME / ".cache/cmux-job", True, "next cmux reload or CI job rebuilds it", depth=2, git_disposable=True, - owner="cmux reload-cloud", rebuild_minutes=30, min_idle_hours=168.0, + owner="cmux reload-cloud", rebuild_minutes=30, min_idle_hours=DEFAULT_AGE_HOURS, mirror_idle_hours=CMUX_JOB_MIRROR_HOURS, platform="macos"), Family("cmux-ghosttykit-cache", HOME / ".cache/cmux/ghosttykit", True, "cmux rebuilds GhosttyKit", depth=1, files=True, - owner="cmux build tooling", rebuild_minutes=5, min_idle_hours=168.0, + owner="cmux build tooling", rebuild_minutes=5, min_idle_hours=DEFAULT_AGE_HOURS, platform="macos"), Family("cua-recordings", HOME / "Library/Application Support/cua-ssh/recordings", True, "the CUA recorder can capture it again", depth=1, files=True, skip=("index.json", "manifest.json", "state.json", ".metadata_never_index", ".lock"), - owner="CUA recorder", rebuild_minutes=2, min_idle_hours=168.0, platform="macos"), + owner="CUA recorder", rebuild_minutes=2, min_idle_hours=DEFAULT_AGE_HOURS, platform="macos"), Family("cua-cache", HOME / "Library/Caches/cmux/cmux-cua", True, "the CUA client rebuilds its frame cache", depth=1, files=True, skip=("index.json", "manifest.json", "state.json", ".metadata_never_index", ".lock"), - owner="cmux CUA", rebuild_minutes=1, min_idle_hours=168.0, platform="macos"), + owner="cmux CUA", rebuild_minutes=1, min_idle_hours=DEFAULT_AGE_HOURS, platform="macos"), ] if (HOME / "Library/Caches" / CMUX_NEXT_CI_ROOT).is_dir(): - roots.append(replace(cmux_next_ci_family(), min_idle_hours=168.0)) + roots.append(replace(cmux_next_ci_family(), min_idle_hours=DEFAULT_AGE_HOURS)) return [f for f in roots if f.root.is_dir()] @@ -7682,7 +7700,7 @@ def _main_impl(argv: list[str] | None = None) -> int: ap.add_argument("--cmux-next-reservation-owner", help="cmux-next drain: exact owner string in the active Glaeda reservation") ap.add_argument("--idle-hours", type=float, default=None, - help="idle window (default 24h, or 6h under pressure)") + help="idle window (default 12h, or 6h under pressure)") ap.add_argument("--family", action="append", help="limit to these family ids") ap.add_argument("--runner-caches", action="store_true", help="use the narrow enrolled-runner account profile (macOS only)") @@ -7936,7 +7954,7 @@ def _main_impl(argv: list[str] | None = None) -> int: maintenance_handle.close() return result stale_marker_released = clear_stale_build_markers(a.receipt) - idle = a.idle_hours if a.idle_hours is not None else IDLE_SWEEP_HOURS if sweep else (6 if under else 24) + idle = a.idle_hours if a.idle_hours is not None else IDLE_SWEEP_HOURS if sweep else (6 if under else DEFAULT_AGE_HOURS) at, known, measured_at = (0.0, {}, {}) if a.refresh or a.no_snapshot else load_snapshot_meta() original_known = set(known) _REFRESH_PROGRESS_KNOWN = dict(known) @@ -8069,8 +8087,9 @@ def _main_impl(argv: list[str] | None = None) -> int: if not RUNNER_CACHE_PROFILE and not a.cmux_next_drain \ and (not a.family or TMP_TEST_RESIDUE_FAMILY in a.family): logical_released += reap_tmpdir_test_residue(a.receipt, halt=halt) - # still under the runner gate's floor after every family: idle runners' whole checkouts, last - if a.pressure and not RUNNER_CACHE_PROFILE and not a.cmux_next_drain \ + # idle runners' whole checkouts, last: stale ones on every pressure or sweep pass, younger ones only while + # still under the runner gate's floor after every family + if (a.pressure or sweep) and not RUNNER_CACHE_PROFILE and not a.cmux_next_drain \ and (not a.family or RUNNER_IDLE_CHECKOUT_FAMILY in a.family): logical_released += evict_idle_runner_checkouts(a.receipt, floor_gib) if last: diff --git a/scripts/test-glaeda-disk.py b/scripts/test-glaeda-disk.py index fdc50559c..1efc6f2fe 100644 --- a/scripts/test-glaeda-disk.py +++ b/scripts/test-glaeda-disk.py @@ -2910,14 +2910,14 @@ def test_apple_generation_stays_while_a_run_holds_the_lock(self) -> None: fd = os.open(state / "build.lock", os.O_RDWR | os.O_CREAT, 0o600) self.addCleanup(os.close, fd) fcntl.flock(fd, fcntl.LOCK_EX) - v = {str(Path(i.path).relative_to(projects)): i.verdict for i in gd.survey([fam], 6, 0)} + v = {str(Path(i.path).relative_to(projects)): i.verdict for i in gd.survey([fam], 24, 0)} self.assertEqual(v["app/.glaeda/apple-build/cache/k1"], "kept") fcntl.flock(fd, fcntl.LOCK_UN) (state / "inflight.json").write_text("{}") - v = {str(Path(i.path).relative_to(projects)): i.verdict for i in gd.survey([fam], 6, 0)} + v = {str(Path(i.path).relative_to(projects)): i.verdict for i in gd.survey([fam], 24, 0)} self.assertEqual(v["app/.glaeda/apple-build/cache/k1"], "kept") (state / "inflight.json").unlink() - v = {str(Path(i.path).relative_to(projects)): i.verdict for i in gd.survey([fam], 6, 0)} + v = {str(Path(i.path).relative_to(projects)): i.verdict for i in gd.survey([fam], 24, 0)} self.assertEqual(v["app/.glaeda/apple-build/cache/k1"], "reclaimable") def test_checkout_builds_ignore_the_callers_git_dir(self) -> None: @@ -3059,7 +3059,7 @@ def not_newest(level, commands=None): def verdicts(hold): with mock.patch.object(gd, "keep_live_app_clones", side_effect=not_newest), \ mock.patch.object(gd, "ACTIVITY_MODE", "pressure"), mock.patch.object(gd, "FLOOR_HOLD", hold): - return {Path(i.path).name: i.verdict for i in gd.survey([fam], 6, 0)} + return {Path(i.path).name: i.verdict for i in gd.survey([fam], 24, 0)} self.assertEqual(verdicts({}), {"code_sign_clone.Old1": "recent", "code_sign_clone.Old2": "recent"}) self.assertEqual(verdicts({dev: 50 * gd.GIB}), {"code_sign_clone.Old1": "reclaimable", "code_sign_clone.Old2": "reclaimable"}) @@ -3729,7 +3729,7 @@ def test_the_per_host_cmux_next_root_goes_last_and_only_under_the_floor(self) -> gd._DISCOVERED.clear() with mock.patch.object(gd, "HOME", home), mock.patch.object(gd, "process_evidence", return_value=([], "")), \ mock.patch.object(gd, "ACTIVITY_MODE", "pressure"), mock.patch.object(gd, "CMUX_NEXT_CI_OVER", gd.GIB): - v = {Path(i.path): i.verdict for i in gd.survey([fam], 6, 0)} + v = {Path(i.path): i.verdict for i in gd.survey([fam], 24, 0)} self.assertEqual(v[host], "recent") self.assertNotIn(host / "Release/xcode-2c2299e9015ce0c8", v) # never a state root of its own # 35 GiB free: under the hold's 50 GiB stop, above the 30 GiB floor. Everything else goes; the host stays @@ -3775,13 +3775,16 @@ def test_the_per_host_cmux_next_root_is_kept_while_its_lock_is_held_and_goes_onc fcntl.flock(build, fcntl.LOCK_EX | fcntl.LOCK_NB) self.assertFalse(gd.remove_cmux_next_host(host)) self.assertTrue((host / "src/.git").is_dir()) - # idle 48 h: an idle sweep (no pressure, no floor) takes it, git clone and all; younger, it stays + # the newest warm compile cache per builder is exempt from age by role: an idle sweep (no pressure, no floor) + # never takes it, however old; a finite cmux_next_ci_host_idle_hours in disk-policy.json ages it again receipt = self.receipt() - for hours, gone in ((47.0, False), (49.0, True)): + for hours, gone, window in ((49.0, False, float("inf")), (500.0, False, float("inf")), + (47.0, False, 48.0), (49.0, True, 48.0)): self._age(host, hours=hours) gd._DISCOVERED.clear() with mock.patch.object(gd, "HOME", home), mock.patch.object(gd, "process_evidence", return_value=([], "")), \ mock.patch.object(gd, "command_lines", return_value=""), \ + mock.patch.dict(gd.POLICY, {"cmux_next_ci_host_idle_hours": window}), \ mock.patch.object(gd, "ACTIVITY_MODE", "sweep"), contextlib.redirect_stdout(io.StringIO()): items = [i for i in gd.survey([fam], 6, 0) if Path(i.path) == host] gd.apply(items, {fam.id: fam}, receipt, None, 6, all_families=[fam]) @@ -3799,7 +3802,7 @@ def test_the_per_host_cmux_next_root_is_kept_while_its_lock_is_held_and_goes_onc with mock.patch.dict(gd.POLICY, {"cmux_next_ci_host_roots": []}): self.assertEqual(gd.cmux_next_ci_host_roots(fam.root), []) self.assertEqual(gd.POLICY_DEFAULTS["cmux_next_ci_host_roots"], ["dev-daily-release/host"]) - self.assertEqual(gd.POLICY_DEFAULTS["cmux_next_ci_host_idle_hours"], 48.0) + self.assertEqual(gd.POLICY_DEFAULTS["cmux_next_ci_host_idle_hours"], float("inf")) def test_the_host_lock_keeps_its_inode_so_a_build_that_opened_it_still_excludes_others(self) -> None: # a build opened host/lock (fd 9) before the removal and flocks it after: a new lock file at the path would @@ -4012,7 +4015,7 @@ def verdicts(over, mode="pressure"): with mock.patch.object(gd, "HOME", home), \ mock.patch.object(gd, "process_evidence", return_value=([], worker)), \ mock.patch.object(gd, "ACTIVITY_MODE", mode), mock.patch.object(gd, "CMUX_NEXT_CI_OVER", over): - return {Path(i.path): (i.verdict, i.reasons) for i in gd.survey([fam], 6, 0)} + return {Path(i.path): (i.verdict, i.reasons) for i in gd.survey([fam], 24, 0)} # within the cap: the generated children alone, each waiting out the 72 h window v = verdicts(0) self.assertEqual(set(v), {s / "Build" for s in st.values()}) @@ -4048,7 +4051,7 @@ def test_cmux_next_ci_cap_keeps_every_in_use_guard(self) -> None: with mock.patch.object(gd, "HOME", home), \ mock.patch.object(gd, "process_evidence", return_value=([os.fspath(st["release-new"])], xcodebuild)), \ mock.patch.object(gd, "ACTIVITY_MODE", "pressure"), mock.patch.object(gd, "CMUX_NEXT_CI_OVER", gd.GIB): - v = {Path(i.path): (i.verdict, i.reasons) for i in gd.survey([fam], 6, 0)} + v = {Path(i.path): (i.verdict, i.reasons) for i in gd.survey([fam], 24, 0)} self.assertEqual(v[st["release-old"]][0], "in-use") self.assertIn("active or ambiguous marker", v[st["release-old"]][1][0]) self.assertEqual(v[st["gone-runner"]][0], "in-use") @@ -4063,7 +4066,7 @@ def test_cmux_next_ci_states_stay_partial_without_enrolled_runners(self) -> None gd._DISCOVERED.clear() with mock.patch.object(gd, "HOME", home), mock.patch.object(gd, "process_evidence", return_value=([], "")), \ mock.patch.object(gd, "ACTIVITY_MODE", "pressure"), mock.patch.object(gd, "CMUX_NEXT_CI_OVER", gd.GIB): - paths = {Path(i.path) for i in gd.survey([fam], 6, 0)} + paths = {Path(i.path) for i in gd.survey([fam], 24, 0)} for key in ("gone-runner", "release-old"): self.assertIn(st[key] / "Build", paths) self.assertNotIn(st[key], paths) @@ -4489,6 +4492,46 @@ def test_idle_runner_checkouts_stay_while_above_the_floor(self) -> None: self.assertEqual(removed, []) self.assertTrue(all(c.is_dir() for c in cos.values())) + def test_stale_runner_checkouts_go_above_the_floor_and_without_one(self) -> None: + """Old runner work dirs go by age: a checkout no job used for DEFAULT_AGE_HOURS goes whatever the free space, + oldest first and every one of them; younger ones stay above the floor.""" + stale = gd.RUNNER_STALE_CHECKOUT_HOURS + self.assertEqual(stale, 12.0) + home, cos = self._idle_checkouts({"actions-runner-glaeda": stale + 1, "actions-runner-glaeda-2": stale + 30, + "actions-runner-glaeda-3": stale - 1}) + gd.process_evidence = lambda: ([], "") + for floor in (30.0, 0.0): + with self.subTest(floor=floor): + patches, removed = self._floor_disk(400.0) # far above the floor and its target + receipt = self.receipt() + with patches[0], patches[1], contextlib.redirect_stdout(io.StringIO()) as out: + self.assertGreater(gd.evict_idle_runner_checkouts(receipt, floor, home), 0) + self.assertEqual(sorted(removed), sorted([str(cos["actions-runner-glaeda-2"]), str(cos["actions-runner-glaeda"])])) + self.assertTrue(cos["actions-runner-glaeda-3"].is_dir()) + self.assertIn("unused for 12 h", out.getvalue()) + recs = [json.loads(line) for line in receipt.read_text().splitlines()] + self.assertEqual({r["reasons"][0] for r in recs}, {"unused for 12 h"}) + for name in ("actions-runner-glaeda", "actions-runner-glaeda-2"): # back for the next floor + (cos[name] / ".git").mkdir(parents=True) + self._age(home / name, hours=stale + 1) + + def test_every_disposable_age_is_twelve_hours(self) -> None: + """Leo, 2026-10-26: "When have we ever had something useful greater than 12 hours?" Only the newest warm + compile cache per builder (the per-host cmux-next root) is exempt, by role.""" + self.assertEqual(gd.DEFAULT_AGE_HOURS, 12.0) + for name in ("KEPT_BUILD_IDLE_HOURS", "IDLE_SWEEP_HOURS", "IDLE_SEED_HOURS", "PARKED_NO_LOG_IDLE_HOURS", + "CMUX_NEXT_CI_MIN_IDLE_HOURS", "CANONICAL_ADMISSION_IDLE_HOURS", "CHECKOUT_BUILD_IDLE_HOURS", + "TMP_TEST_RESIDUE_IDLE_HOURS", "RUNNER_STALE_CHECKOUT_HOURS"): + with self.subTest(name): + self.assertEqual(getattr(gd, name), 12.0) + self.assertEqual(gd.RUNTIME_IDLE_DAYS * 24, 12.0) + for key in ("soft_idle_hours", "stale_marker_max_hours"): + self.assertEqual(gd.POLICY_DEFAULTS[key], 12.0, key) + self.assertEqual(gd.POLICY_DEFAULTS["cmux_next_ci_host_idle_hours"], float("inf")) + for fam in [*gd.default_families(), *gd.runner_cache_families()]: + with self.subTest(fam.id): + self.assertLessEqual(fam.min_idle_hours, 12.0) + def test_idle_runner_checkouts_go_between_the_floor_and_the_gates_resume_mark(self) -> None: # the gate holds until floor + 2 GiB; at 31 GiB it still holds, so the last resort still acts home, cos = self._idle_checkouts({"actions-runner-glaeda": 5, "actions-runner-glaeda-2": 5}) From 200d454b736889408a7e476c53f937f6a47e7d42 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 26 Oct 2026 14:23:20 -0700 Subject: [PATCH 04/28] test-glaeda-disk: age-dependent cases follow DEFAULT_AGE_HOURS Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- scripts/glaeda-disk | 1 - scripts/test-glaeda-disk.py | 15 ++++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/glaeda-disk b/scripts/glaeda-disk index bb1d13a82..80350ce8d 100755 --- a/scripts/glaeda-disk +++ b/scripts/glaeda-disk @@ -619,7 +619,6 @@ CMUX_NEXT_DRAIN_HOOK_SHA256 = frozenset({ "d5e73be990d67d9499ff352d01fda2e1392a6fd257a57770b23008b3bf55fb59", "0c2c8880e3b17af8324f8304121c4159bee8ade5c31496b9af713586cb90bcfc", "fca8abb34684680ad0344a7a82b0ee8b2b476f6490407de27eb410904f5fac91", - "6bdc89db3e25e5fa40321e5d3066d7d5cf4ba7354c6915687cd76f118d7dfec5", }) CMUX_NEXT_CI_PRESERVED_NAMES = frozenset({ "SourcePackages", "ModuleCache.noindex", "SDKStatCaches.noindex", diff --git a/scripts/test-glaeda-disk.py b/scripts/test-glaeda-disk.py index 1efc6f2fe..ec14f6abb 100644 --- a/scripts/test-glaeda-disk.py +++ b/scripts/test-glaeda-disk.py @@ -233,7 +233,7 @@ def test_runner_cache_profile_is_allowlisted_and_bounded(self) -> None: {"xcode-derived-data", "cmux-job-cache", "cmux-ghosttykit-cache", "cua-recordings", "cua-cache"}, ) - self.assertTrue(all(family.min_idle_hours == 168.0 for family in families)) + self.assertTrue(all(family.min_idle_hours == gd.DEFAULT_AGE_HOURS for family in families)) self.assertTrue(gd.runner_profile_roots_safe(families)) measured = gd.fixed_measurement_roots() self.assertEqual(set(measured), {family.root for family in families}) @@ -1620,7 +1620,7 @@ def test_default_families_delineate_cua_recordings_and_cache(self) -> None: gd.HOME / "Library/Application Support/cua-ssh/recordings") self.assertEqual(fams["cua-cache"].root, gd.HOME / "Library/Caches/cmux/cmux-cua") self.assertTrue(fams["cua-recordings"].reclaimable) - self.assertEqual(fams["cua-recordings"].min_idle_hours, 24.0) + self.assertEqual(fams["cua-recordings"].min_idle_hours, gd.DEFAULT_AGE_HOURS) self.assertEqual(fams["cua-cache"].owner, "cmux CUA") finally: gd.HOME, gd.FLEET_ROOT, gd.DARWIN = saved @@ -1724,7 +1724,7 @@ def test_cmux_next_ci_keeps_recent_and_marked_state(self) -> None: def test_cmux_next_ci_idle_floor_survives_pressure_mode(self) -> None: root = self.root / "Library/Caches/cmux-next-ci" - build = make(root / "cmux-scheme/runner-a/Build", age_hours=48) + build = make(root / "cmux-scheme/runner-a/Build", age_hours=gd.CMUX_NEXT_CI_MIN_IDLE_HOURS - 1) fam = gd.cmux_next_ci_family(root) with mock.patch.object(gd, "ACTIVITY_MODE", "pressure"), \ mock.patch.object(gd, "process_evidence", return_value=([], "")): @@ -2017,7 +2017,7 @@ def test_canonical_admission_gives_way_under_the_runner_floor_only_while_no_job_ for k in (1, 2, 3): (capacity / f"root-{k}.token").touch() roots = {k: tmp / ("cmux-ci" if k == 1 else f"cmux-ci-{k}") for k in (1, 2, 3)} - for k, hours in ((1, 9.3), (2, 20.9), (3, 0.5)): + for k, hours in ((1, 9.3), (2, 10.9), (3, 0.5)): # under the 12 h window make(roots[k] / "derived-data-compile-admission", age_hours=hours) make(roots[k] / "src", age_hours=hours) job = open(capacity / "root-3.token") @@ -2031,7 +2031,7 @@ def verdicts(hold): fams = gd.canonical_admission_families(tmp) return fams, {os.path.relpath(i.path, tmp): i.verdict for i in gd.survey(fams, 6, 0)} _, above = verdicts({}) - self.assertEqual(set(above.values()), {"recent", "in-use"}) # above the floor the 24 h window stands + self.assertEqual(set(above.values()), {"recent", "in-use"}) # above the floor the 12 h window stands fams, under = verdicts({dev: 50 * gd.GIB}) self.assertEqual(under, {"cmux-ci/derived-data-compile-admission": "reclaimable", "cmux-ci-2/derived-data-compile-admission": "reclaimable", @@ -3857,6 +3857,7 @@ def test_a_finished_host_aside_counts_against_the_cap(self) -> None: item = gd.Item(fam.id, os.fspath(host), 5 * 1024 * 1024, 49, "reclaimable", []) with mock.patch.object(gd, "HOME", home), mock.patch.object(gd, "process_evidence", return_value=([], "")), \ mock.patch.object(gd, "command_lines", return_value=""), \ + mock.patch.dict(gd.POLICY, {"cmux_next_ci_host_idle_hours": 48.0}), \ mock.patch.object(gd, "ACTIVITY_MODE", "sweep"), contextlib.redirect_stdout(io.StringIO()): released = gd.apply([item], {fam.id: fam}, receipt, None, 6, all_families=[fam], taken=taken) self.assertEqual(released, 2 * 1024 * 1024) @@ -4735,7 +4736,7 @@ def test_tmpdir_test_residue_reaps_only_old_named_test_scratch(self) -> None: self.assertEqual((rec["family"], rec["outcome"], rec["items"]), (gd.TMP_TEST_RESIDUE_FAMILY, "reclaimed", 4)) self.assertEqual(rec["by_prefix"], {"cmux-cli-socket": 1, "git-pr": 1, "TemporaryDirectory": 1, "cmux-shortcut-routing": 1}) - self.assertEqual(rec["kept"], {"a symlink": 1, "changed within 48h": 1}) # recent tops are never walked + self.assertEqual(rec["kept"], {"a symlink": 1, f"changed within {gd.TMP_TEST_RESIDUE_IDLE_HOURS:g}h": 1}) # recent tops are never walked def test_tmpdir_test_residue_vetoes_open_and_named_entries(self) -> None: tmp, e = self._tmp_residue() @@ -5387,7 +5388,7 @@ def test_idle_sweep_in_main(self) -> None: def test_simulator_runtimes_go_only_when_nothing_uses_them(self) -> None: now = time.time() old = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(now - 30 * 86400)) - new = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(now - 86400)) + new = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(now - 6 * 3600)) # inside RUNTIME_IDLE_DAYS def rt(version, used, deletable=True): return {"runtimeIdentifier": f"ios-{version}", "platformIdentifier": "iphonesimulator", From 8dc16ac5f73cd37b3b0bf0541039afbafe213843 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 26 Oct 2026 14:44:15 -0700 Subject: [PATCH 05/28] glaeda-disk: a stale runner checkout above the floor keeps git's vetoes (no lock, no commit only here) Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- scripts/glaeda-disk | 33 +++++++++++++++++++++++++- scripts/test-glaeda-disk.py | 47 +++++++++++++++++++++++++++++-------- 2 files changed, 69 insertions(+), 11 deletions(-) diff --git a/scripts/glaeda-disk b/scripts/glaeda-disk index 80350ce8d..00388b3e4 100755 --- a/scripts/glaeda-disk +++ b/scripts/glaeda-disk @@ -1940,6 +1940,34 @@ def runner_checkout_last_use(runner: Path, checkout: Path) -> float: return max(times) +GIT_LOCK_NAMES = ("index.lock", "HEAD.lock", "shallow.lock", "packed-refs.lock", "config.lock", "ORIG_HEAD.lock") + + +def runner_checkout_git_veto(checkout: Path) -> str: + """Why a stale runner checkout must stay for git's sake, or "": git holds a lock in it, or a commit there (HEAD, a + local branch or tag, the stash, any worktree's HEAD) is on no remote-tracking ref. Untracked and ignored files are + a job's build output and go with it. Any doubt (git fails or times out) keeps it.""" + dotgit = checkout / ".git" + if dotgit.is_symlink() or not dotgit.is_dir(): + return "not a plain checkout (.git is not a directory)" + with contextlib.suppress(OSError): + for root, dirs, files in os.walk(dotgit): + dirs[:] = [d for d in dirs if d not in ("objects", "lfs", "logs")] + if any(f in GIT_LOCK_NAMES or (f.endswith(".lock") and "refs" in Path(root).relative_to(dotgit).parts) + for f in files): + return "git holds a lock in it" + env = {**os.environ, "GIT_OPTIONAL_LOCKS": "0"} + try: + tips = subprocess.run(["git", "-C", os.fspath(checkout), "for-each-ref", "--format=%(objectname)", + "refs/heads", "refs/tags", "refs/stash"], capture_output=True, text=True, timeout=60, + env=env, check=True).stdout.split() + local = subprocess.run(["git", "-C", os.fspath(checkout), "rev-list", "--max-count=1", "HEAD", *tips, + "--not", "--remotes"], capture_output=True, text=True, timeout=120, env=env, check=True) + except (OSError, subprocess.SubprocessError): + return "git could not say whether a commit is only here" + return "a commit here is on no remote-tracking ref" if local.stdout.strip() else "" + + def idle_runner_checkout_veto(runner: Path, checkout: Path, opened: frozenset[str], cmds: str) -> str: """Why an idle runner's checkout must stay, or "" when it may go.""" path = os.fspath(checkout) @@ -1996,9 +2024,12 @@ def evict_idle_runner_checkouts(receipt: Path, floor_gib: float, home: Path | No print(f"refusing to delete idle runner checkouts: {e}", file=sys.stderr) break why = idle_runner_checkout_veto(runner, checkout, open_index(opened), cmds) + if not why and stale and not under: # above the floor only what git can lose nothing of goes + why = runner_checkout_git_veto(checkout) size = 0 if why: - outcome = "changed:modified" if why.startswith("used within") else "changed:in-use" + outcome = ("changed:modified" if why.startswith("used within") + else "blocked:git" if why.startswith(("git", "a commit", "not a plain")) else "changed:in-use") else: try: before = free_bytes(home)[0] diff --git a/scripts/test-glaeda-disk.py b/scripts/test-glaeda-disk.py index ec14f6abb..063d7d922 100644 --- a/scripts/test-glaeda-disk.py +++ b/scripts/test-glaeda-disk.py @@ -4493,28 +4493,55 @@ def test_idle_runner_checkouts_stay_while_above_the_floor(self) -> None: self.assertEqual(removed, []) self.assertTrue(all(c.is_dir() for c in cos.values())) + def _git_checkout(self, co: Path, pushed: bool = True) -> None: + """A real checkout at co whose HEAD is on origin/main (pushed) or one commit past it.""" + env = {"PATH": "/usr/bin:/bin:/opt/homebrew/bin", "HOME": os.fspath(self.root), "GIT_AUTHOR_NAME": "t", + "GIT_AUTHOR_EMAIL": "t@x", "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@x"} + + def git(*args: str) -> None: + subprocess.run(["git", "-C", os.fspath(co), *args], env=env, check=True, capture_output=True) + if (co / ".git").exists(): + shutil.rmtree(co / ".git") + co.mkdir(parents=True, exist_ok=True) + git("init", "-q") + git("commit", "-q", "--allow-empty", "-m", "base") + git("update-ref", "refs/remotes/origin/main", "HEAD") + git("checkout", "-q", "--detach") + if not pushed: + git("commit", "-q", "--allow-empty", "-m", "local only") + (co / "build-output.o").write_text("x") # untracked build output never keeps a checkout + def test_stale_runner_checkouts_go_above_the_floor_and_without_one(self) -> None: """Old runner work dirs go by age: a checkout no job used for DEFAULT_AGE_HOURS goes whatever the free space, - oldest first and every one of them; younger ones stay above the floor.""" + unless git would lose something (a commit on no remote-tracking ref) or holds a lock; younger ones stay.""" stale = gd.RUNNER_STALE_CHECKOUT_HOURS self.assertEqual(stale, 12.0) - home, cos = self._idle_checkouts({"actions-runner-glaeda": stale + 1, "actions-runner-glaeda-2": stale + 30, - "actions-runner-glaeda-3": stale - 1}) + names = {"actions-runner-glaeda": stale + 1, "actions-runner-glaeda-2": stale + 30, + "actions-runner-glaeda-3": stale - 1, "actions-runner-glaeda-4": stale + 2, + "actions-runner-glaeda-5": stale + 3} + home, cos = self._idle_checkouts(names) gd.process_evidence = lambda: ([], "") for floor in (30.0, 0.0): with self.subTest(floor=floor): + for name, age in names.items(): + self._git_checkout(cos[name], pushed=name != "actions-runner-glaeda-4") + if name == "actions-runner-glaeda-5": + (cos[name] / ".git/index.lock").write_text("") + self._age(home / name, hours=age) patches, removed = self._floor_disk(400.0) # far above the floor and its target receipt = self.receipt() with patches[0], patches[1], contextlib.redirect_stdout(io.StringIO()) as out: self.assertGreater(gd.evict_idle_runner_checkouts(receipt, floor, home), 0) - self.assertEqual(sorted(removed), sorted([str(cos["actions-runner-glaeda-2"]), str(cos["actions-runner-glaeda"])])) - self.assertTrue(cos["actions-runner-glaeda-3"].is_dir()) + self.assertEqual(sorted(removed), sorted([str(cos["actions-runner-glaeda-2"]), + str(cos["actions-runner-glaeda"])])) + for name in ("actions-runner-glaeda-3", "actions-runner-glaeda-4", "actions-runner-glaeda-5"): + self.assertTrue((cos[name] / ".git").is_dir(), name) self.assertIn("unused for 12 h", out.getvalue()) - recs = [json.loads(line) for line in receipt.read_text().splitlines()] - self.assertEqual({r["reasons"][0] for r in recs}, {"unused for 12 h"}) - for name in ("actions-runner-glaeda", "actions-runner-glaeda-2"): # back for the next floor - (cos[name] / ".git").mkdir(parents=True) - self._age(home / name, hours=stale + 1) + why = {Path(r["path"]).parts[-4]: (r["outcome"], r["reasons"][0]) + for r in map(json.loads, receipt.read_text().splitlines())} + self.assertEqual(why["actions-runner-glaeda-4"], ("blocked:git", "a commit here is on no remote-tracking ref")) + self.assertEqual(why["actions-runner-glaeda-5"], ("blocked:git", "git holds a lock in it")) + self.assertEqual(why["actions-runner-glaeda"], ("reclaimed", "unused for 12 h")) def test_every_disposable_age_is_twelve_hours(self) -> None: """Leo, 2026-10-26: "When have we ever had something useful greater than 12 hours?" Only the newest warm From 25efbb012fa9925c5e6d092612e0f55009a1c464 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 26 Oct 2026 14:48:36 -0700 Subject: [PATCH 06/28] glaeda-disk: a reserved host keeps the ages before 12 h until its reservation lapses Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- scripts/glaeda-disk | 20 ++++++++++++++++++++ scripts/test-glaeda-disk.py | 19 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/scripts/glaeda-disk b/scripts/glaeda-disk index 00388b3e4..1ffc1b057 100755 --- a/scripts/glaeda-disk +++ b/scripts/glaeda-disk @@ -5318,6 +5318,25 @@ def host_reservation(now: float | None = None, path: Path | None = None, names: return "" +# The ages before DEFAULT_AGE_HOURS (2026-10-26). A host a fleet reservation holds keeps them (and gets only +# floor-pressure eviction, _main_impl): its holder's idle state is theirs until the reservation lapses, when the 12 h +# ages apply by themselves. Family minimums that read DEFAULT_AGE_HOURS take its legacy value, 24 h. +LEGACY_AGES = {"DEFAULT_AGE_HOURS": 24.0, "KEPT_BUILD_IDLE_HOURS": 24.0, "IDLE_SWEEP_HOURS": 24.0, + "PARKED_NO_LOG_IDLE_HOURS": 168.0, "CMUX_NEXT_CI_MIN_IDLE_HOURS": 72.0, + "CANONICAL_ADMISSION_IDLE_HOURS": 24.0, "IDLE_SEED_HOURS": 12.0, "RUNTIME_IDLE_DAYS": 14.0, + "CHECKOUT_BUILD_IDLE_HOURS": 24.0, "TMP_TEST_RESIDUE_IDLE_HOURS": 48.0, + "RUNNER_STALE_CHECKOUT_HOURS": float("inf")} +LEGACY_POLICY = {"soft_idle_hours": 24.0, "stale_marker_max_hours": 48.0, "cmux_next_ci_host_idle_hours": 48.0} + + +def keep_legacy_ages() -> None: + """Put back the pre-12 h ages for this run (a reserved host). POLICY keys a disk-policy.json sets stay its own.""" + globals().update(LEGACY_AGES) + for key, value in LEGACY_POLICY.items(): + if POLICY.get(key) == POLICY_DEFAULTS.get(key): + POLICY[key] = value + + def filesystems(fams: list[Family], low: str, target: str, tmpfs_low: str = "75%", tmpfs_target: str = "90%", floor_gib: float = 0.0) -> dict[int, Fs]: out: dict[int, Fs] = {} @@ -7857,6 +7876,7 @@ def _main_impl(argv: list[str] | None = None) -> int: floor_gib = max(runner_floor_gib(), a.floor_gib if math.isfinite(a.floor_gib) else 0.0) if reserved: RUN["reserved"] = reserved + keep_legacy_ages() a.idle = False # no idle sweep on a reserved host if not a.pressure or floor_gib <= 0: print(f"{reserved}: only floor-pressure eviction runs here" diff --git a/scripts/test-glaeda-disk.py b/scripts/test-glaeda-disk.py index 063d7d922..ed113c97e 100644 --- a/scripts/test-glaeda-disk.py +++ b/scripts/test-glaeda-disk.py @@ -6308,6 +6308,18 @@ def test_the_host_local_reservation_counts(self) -> None: "until": now - 1})) self.assertEqual(self.reserved({"cmux31"}), "") + def test_a_reserved_host_keeps_the_ages_before_twelve_hours(self) -> None: + names = [*gd.LEGACY_AGES] + saved = {n: getattr(gd, n) for n in names} + self.addCleanup(lambda: [setattr(gd, n, v) for n, v in saved.items()]) + with mock.patch.dict(gd.POLICY, {"stale_marker_max_hours": 5.0}): + gd.keep_legacy_ages() + self.assertEqual(gd.POLICY["soft_idle_hours"], 24.0) + self.assertEqual(gd.POLICY["cmux_next_ci_host_idle_hours"], 48.0) + self.assertEqual(gd.POLICY["stale_marker_max_hours"], 5.0, "a disk-policy.json value stays") + self.assertEqual((gd.DEFAULT_AGE_HOURS, gd.CMUX_NEXT_CI_MIN_IDLE_HOURS, gd.RUNTIME_IDLE_DAYS), (24.0, 72.0, 14.0)) + self.assertEqual(gd.RUNNER_STALE_CHECKOUT_HOURS, float("inf"), "no above-floor checkout eviction") + def test_the_shipped_list_parses_and_matches_the_registry_where_it_is_checked_out(self) -> None: shipped = json.loads((ROOT / "config/fleet-reservations.json").read_text()) for e in shipped["reservations"]: @@ -6324,6 +6336,9 @@ def test_the_shipped_list_parses_and_matches_the_registry_where_it_is_checked_ou def run_disk(self, *args: str, reserved: str, floor: float, free_gib: float): """glaeda-disk's main on a fake fleet mini; returns (exit code, output, the families' low/target it used).""" seen, run = {}, {} + saved = {n: getattr(gd, n) for n in getattr(gd, "LEGACY_AGES", {})} + self.addCleanup(lambda: [setattr(gd, n, v) for n, v in saved.items()]) + self.addCleanup(lambda policy=dict(gd.POLICY): (gd.POLICY.clear(), gd.POLICY.update(policy))) def filesystems(fams, low, target, *rest): seen.update(low=low, target=target) @@ -6357,7 +6372,11 @@ def test_a_reserved_host_runs_only_floor_pressure_to_the_floor_plus_the_margin(s self.assertEqual(seen, {"low": "30", "target": f"{30 + margin:g}"}) self.assertEqual(run["reserved"], why) # an unreserved host keeps its policy + self.assertEqual(gd.DEFAULT_AGE_HOURS, 24.0, "the reserved run kept the legacy ages") + for n, v in {n: 12.0 for n in ("DEFAULT_AGE_HOURS",)}.items(): + setattr(gd, n, v) code, out, seen, run = self.run_disk("--pressure", reserved="", floor=30.0, free_gib=50) + self.assertEqual(gd.DEFAULT_AGE_HOURS, 12.0) self.assertEqual(seen, {"low": gd.POLICY["low"], "target": gd.POLICY["target"]}) From 3ee1a9e9cc9a7725e30adc7a1e7d74706eedb491 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 26 Oct 2026 14:52:58 -0700 Subject: [PATCH 07/28] glaeda-disk: a reserved host's families are built with the legacy ages Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- scripts/glaeda-disk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/glaeda-disk b/scripts/glaeda-disk index 1ffc1b057..53b10e76d 100755 --- a/scripts/glaeda-disk +++ b/scripts/glaeda-disk @@ -7831,6 +7831,8 @@ def _main_impl(argv: list[str] | None = None) -> int: # before any family is built: a reserved host's families keep its holder's ages (keep_legacy_ages, once 12 h lands) reserved = host_reservation() if role == "fleet-mini" and DARWIN and a.apply and not a.runner_caches else "" + if reserved: + keep_legacy_ages() global RUNNER_CACHE_PROFILE RUNNER_CACHE_PROFILE = bool(a.runner_caches) fams = runner_cache_families() if RUNNER_CACHE_PROFILE else default_families(role) @@ -7876,7 +7878,6 @@ def _main_impl(argv: list[str] | None = None) -> int: floor_gib = max(runner_floor_gib(), a.floor_gib if math.isfinite(a.floor_gib) else 0.0) if reserved: RUN["reserved"] = reserved - keep_legacy_ages() a.idle = False # no idle sweep on a reserved host if not a.pressure or floor_gib <= 0: print(f"{reserved}: only floor-pressure eviction runs here" From 52cde724a9a536af66030c684d391c56e39aa2e7 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 26 Oct 2026 14:21:44 -0700 Subject: [PATCH 08/28] glaeda-disk: mirrors go at 12 h, their unpushed commits and local-only refs bundled to ~/archive first Before a mirror family's item (cmux-job-cache, the hq build cache) goes, archive_mirror finds every repository it takes along (checkouts, bare repos, a linked worktree's repository inside it, submodule stores, embedded submodules) and writes each one's commits no remote holds, plus branches and tags only this host has, into git bundles under ~/archive/glaeda-disk, named with a 30-day expiry and pruned by that name. Each bundle passes `git bundle verify` and lists every ref it must hold before the tree is deleted; a failure keeps the tree, records blocked:bundle-failed in the receipt and run line, and alerts on stderr. CMUX_JOB_MIRROR_HOURS drops from 168 to DEFAULT_AGE_HOURS (12). Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- scripts/glaeda-disk | 220 +++++++++++++++++++++++++++++++++--- scripts/test-glaeda-disk.py | 132 +++++++++++++++++++--- 2 files changed, 324 insertions(+), 28 deletions(-) diff --git a/scripts/glaeda-disk b/scripts/glaeda-disk index 53b10e76d..1d842843f 100755 --- a/scripts/glaeda-disk +++ b/scripts/glaeda-disk @@ -161,6 +161,12 @@ commits fetched by id are what the server confirms. A linked worktree must be un HEAD on a branch, tag or remote ref of its repository, which survives, and no commit only its own reflog holds. Ignored files go with it: in a temp-dir checkout they are build output or copies. Nested or unverifiable checkouts stay protected. + +A mirror family's item (cmux-job-cache, the hq build cache: mirror_idle_hours) idle DEFAULT_AGE_HOURS goes +whole, but first archive_mirror writes every commit no remote holds, and every branch or tag only this host +has, into git bundles under ~/archive/glaeda-disk (named -.expires-.bundle, pruned by that +name after MIRROR_ARCHIVE_DAYS) and checks each with `git bundle verify`. A bundle that fails keeps the tree: +the receipt says blocked:bundle-failed, the run line lists it, and stderr alerts. """ from __future__ import annotations @@ -2341,9 +2347,10 @@ HQ_CACHE_FAMILY = "hq-build-fleet-cache" # laptop, which keeps the edits. So once a unit sat idle a week, no git check keeps it. Before that, a clone # still has to lose nothing. Every git check kept them: 107 GiB idle since Sep 28-Oct 2 on eight minis read # "git-checkout (checkout below the top level)" while glaeda-disk reported 0.0 G reclaimable (2026-10-11). -# Not DEFAULT_AGE_HOURS: past this window a checkout counts as a mirror and goes without the git-loss check, so a -# commit only it holds would be lost unarchived; the disposable output inside it goes at DEFAULT_AGE_HOURS. -CMUX_JOB_MIRROR_HOURS = 168.0 +# A week of whole checkouts filled the minis; what is worth keeping is a commit or ref no remote has, so past this +# window archive_mirror bundles those into ~/archive and the tree goes (Leo, 2026-10-26: "When have we ever had +# something useful greater than 12 hours?"). +CMUX_JOB_MIRROR_HOURS = DEFAULT_AGE_HOURS def transcript_candidates(root: Path, archives: bool = False) -> list[Path]: @@ -2371,7 +2378,7 @@ def transcript_candidates(root: Path, archives: bool = False) -> list[Path]: # labelled the lent minis' live cmux-workspaces "RETIRED" (cmux12s, 2026-10-11). # A retired worker's checkouts are mirrors as a reload unit's are: cmux-workspaces//src is a clone the # worker synced from GitHub, swiftpm holds SwiftPM's own clones, and no person edits either. So once an entry -# sat idle a week, no git check keeps it, and a tree too deep to search (cmux-tui-target, a cargo target dir) +# sat idle past CMUX_JOB_MIRROR_HOURS, no git check keeps it (archive_mirror bundles what is unpushed), and a tree too deep to search (cmux-tui-target, a cargo target dir) # is not kept as "unchecked" either. A fleet lease or running job that names an entry still keeps it, as in # fleet-artifacts over the same directory. cmux14 sat at the 30 GiB floor with 0.0G reclaimable while # cmux-workspaces (11.6 GiB, idle 190 h), cmux-tui-target (4.2 GiB, 289 h) and swiftpm (3.9 GiB, 310 h) of @@ -3492,9 +3499,15 @@ def derived_data(fam: Family, path: Path) -> bool: or (fam.id == "cmux-job-cache" and (path.name == "DerivedData" or path.parent.name == "SourcePackages"))) +def mirror_due(fam: Family, path: Path, idle_hours: float) -> bool: + """A mirror family's item idle past its mirror age: it goes once archive_mirror has bundled what is unpushed.""" + return (fam.mirror_idle_hours is not None and idle_hours >= fam.mirror_idle_hours + and not derived_data(fam, path)) + + def git_exempt(fam: Family, path: Path, idle_hours: float) -> bool: - """No git check applies: build output (derived_data), or a mirror family's item idle past its mirror age.""" - return derived_data(fam, path) or (fam.mirror_idle_hours is not None and idle_hours >= fam.mirror_idle_hours) + """No git check applies: build output (derived_data), or a mirror (mirror_due), whose apply bundles first.""" + return derived_data(fam, path) or mirror_due(fam, path, idle_hours) # A deletion interrupted after remove()'s rename (the process killed, the host rebooted) leaves the tree under @@ -4244,13 +4257,13 @@ def git_loss(path: Path) -> str: def _git_dir_run(gitdir: Path, *args: str, timeout: int = 60, - network: bool = False) -> str | None: + network: bool = False, stdin: str | None = None) -> str | None: env = {**os.environ, "GIT_OPTIONAL_LOCKS": "0", "GIT_TERMINAL_PROMPT": "0"} if network and "GIT_SSH_COMMAND" not in env: env["GIT_SSH_COMMAND"] = "ssh -o BatchMode=yes -o ConnectTimeout=10" try: r = subprocess.run(["git", "--git-dir", str(gitdir), *args], capture_output=True, - text=True, timeout=timeout, env=env) + text=True, timeout=timeout, env=env, input=stdin) except (OSError, subprocess.TimeoutExpired): return None return r.stdout if r.returncode == 0 else None @@ -4373,6 +4386,176 @@ def repo_loss(gitdir: Path) -> str: return "" +# archive_mirror's bundles: only files this tool named so are ever pruned, only in its own directory +MIRROR_ARCHIVE_DAYS = 30 +MIRROR_BUNDLE_RE = re.compile(r"\d{8}T\d{6}-[A-Za-z0-9._-]+\.expires-(\d{4}-\d{2}-\d{2})\.bundle") +ARCHIVE_REF = "refs/glaeda-archive/" # a commit only a reflog or HEAD holds is named for the bundle meanwhile + + +def mirror_archive() -> Path: + return HOME / "archive" / "glaeda-disk" + + +def prune_mirror_archive(today: str | None = None) -> int: + """Delete this tool's bundles whose name's expiry date has passed; nothing else in ~/archive.""" + today = today or time.strftime("%Y-%m-%d") + root, gone = mirror_archive(), 0 + if root.is_symlink() or not root.is_dir(): + return 0 + for e in _entries(root): + m = MIRROR_BUNDLE_RE.fullmatch(e.name) + if m and m.group(1) < today and e.is_file(follow_symlinks=False): + with contextlib.suppress(OSError): + os.unlink(e.path) + gone += 1 + return gone + + +def _write_bundle(gitdir: Path, name: str, revs: list[str], heads: set[str], written: list[Path]) -> str: + """`git bundle create` from rev-list arguments, checked by `git bundle verify` and its heads, then moved + to its archive name (appended to written). "" or why it failed.""" + expires = time.strftime("%Y-%m-%d", time.localtime(time.time() + MIRROR_ARCHIVE_DAYS * 86400)) + dest = mirror_archive() / f"{name}.expires-{expires}.bundle" + tmp = dest.with_name(f".{dest.name}.{os.getpid()}.tmp") + try: + dest.parent.mkdir(parents=True, exist_ok=True) + if _git_dir_run(gitdir, "bundle", "create", str(tmp), "--stdin", stdin="\n".join(revs) + "\n", + timeout=600) is None or _git_dir_run(gitdir, "bundle", "verify", str(tmp), timeout=600) is None: + return f"git bundle create or verify failed for {gitdir}" + listed = _git_dir_run(gitdir, "bundle", "list-heads", str(tmp)) + if listed is None or not heads <= {ln.split(" ", 1)[-1] for ln in listed.splitlines()}: + return f"bundle of {gitdir} lacks a ref" + os.replace(tmp, dest) + written.append(dest) + return "" + except OSError as e: + return f"bundle of {gitdir}: {e.__class__.__name__}" + finally: + with contextlib.suppress(OSError): + tmp.unlink(missing_ok=True) + + +def bundle_unpushed(gitdir: Path, stem: str, written: list[Path]) -> str: + """Bundle what deleting this repository loses: commits no remote holds (found as repo_loss finds them, + with the heads and tags the remotes advertise counted as held) and every branch or tag only it has. + "" or why it failed; the bundles go to written.""" + tips = local_tips(gitdir) + for wt in [p for p in (gitdir / "worktrees").glob("*") if p.is_dir()] if tips is not None else []: + own = local_tips(wt, worktree_only=True) + tips = None if own is None else sorted({*tips, *own}) + remotes = network_remotes(gitdir) + refs = _git_dir_run(gitdir, "for-each-ref", "--format=%(refname)%09%(objectname)%09%(*objectname)") + if tips is None or remotes is None or refs is None: + return f"unreadable refs or remotes in {gitdir}" + local: dict[str, tuple[str, str]] = {} # refname -> (object, commit) + held: dict[str, set[str]] = {} # refname (as the remote names it) -> objects a remote holds there + for line in refs.splitlines(): + name, obj, peeled = (line.split("\t") + ["", ""])[:3] + r = next((r for r in remotes if name.startswith(f"refs/remotes/{r}/")), None) + if r is not None: + held.setdefault("refs/heads/" + name[len(f"refs/remotes/{r}/"):], set()).add(obj) + elif not name.startswith("refs/remotes/"): + local[name] = (obj, peeled or obj) + answered = not remotes + for r in remotes: + if (out := _git_dir_run(gitdir, "ls-remote", "--heads", "--tags", r, network=True, timeout=120)) is not None: + answered = True + for ln in out.splitlines(): + obj, _, name = ln.partition("\t") + held.setdefault(name.removesuffix("^{}"), set()).add(obj) + if not answered: + return f"no remote of {gitdir} answered" + objs = sorted({o for s in held.values() for o in s}) + have = _git_dir_run(gitdir, "cat-file", "--batch-check=%(objectname)", stdin="\n".join(objs) + "\n") \ + if objs else "" + if have is None: + return f"unreadable objects in {gitdir}" + known = [ln for ln in have.splitlines() if not ln.endswith(" missing")] + out = _git_dir_run(gitdir, "rev-list", "--stdin", stdin="\n".join([*tips, "--not", *known]) + "\n", + timeout=600) + if out is None: + return f"unreadable history in {gitdir}" + unpushed = set(out.split()) + loose = [t for t in tips if t in unpushed] + if loose and len(loose) <= 64 and remote_has(gitdir, loose): + unpushed, loose = set(), [] + # the refs to keep: every local one on an unpushed commit, and every branch or tag the remotes lack + keep = {n for n, (_, c) in local.items() if c in unpushed} + only_here = sorted(n for n, (o, _) in local.items() if n.startswith(("refs/heads/", "refs/tags/")) + and n not in keep and o not in held.get(n, ())) + loose = sorted(set(loose) - {local[n][1] for n in keep}) + if loose and _git_dir_run(gitdir, "update-ref", "--stdin", stdin="".join( + f"create {ARCHIVE_REF}{t} {t}\n" for t in loose)) is None: + return f"cannot name the loose commits in {gitdir}" + try: + heads = keep | {ARCHIVE_REF + t for t in loose} + if heads: + if (why := _write_bundle(gitdir, stem, [*sorted(heads), "--not", *known], heads, written)): + return why + for n in only_here: # its commit is on a remote: the bundle holds that commit and the name + parents = _git_dir_run(gitdir, "rev-parse", f"{local[n][1]}^@") + if parents is None or (why := _write_bundle( + gitdir, f"{stem}-{re.sub(r'[^A-Za-z0-9._-]+', '_', n)}", [n, "--not", *parents.split()], {n}, written)): + return why or f"unreadable {n} in {gitdir}" + return "" + finally: + if loose: + _git_dir_run(gitdir, "update-ref", "--stdin", stdin="".join( + f"delete {ARCHIVE_REF}{t}\n" for t in loose)) + + +def archive_mirror(path: Path) -> tuple[str, list[Path]]: + """Before a mirror item goes: bundle_unpushed for every repository it takes along, found as _nested_loss + finds them (checkouts, bare repositories, a linked worktree's repository inside it, submodule stores and + embedded submodules). A tree whose repositories cannot all be found or bundled is kept. + ("" or why it must stay, the bundles written).""" + roots = nested_checkouts(path) + if roots is None: + return "too deep or large to find every checkout", [] + inside = os.path.realpath(path) + "/" + gitdirs: dict[str, Path] = {} + for root in roots: + where, dotgit = root.relative_to(path), root / ".git" + if not os.path.lexists(dotgit): + gitdirs[os.path.realpath(root)] = root + elif dotgit.is_symlink(): + return f"{where}: .git is a symlink", [] + elif dotgit.is_file(): # a linked worktree: bundle its repository when that goes too + common, own = git_common_dir(root), _git_dir_run(dotgit, "rev-parse", "--absolute-git-dir") + if not common or own is None: + return f"{where}: unreadable worktree", [] + if (os.path.realpath(common) + "/").startswith(inside): + gitdirs[os.path.realpath(common)] = Path(common) + continue + tips = local_tips(Path(own.strip()), worktree_only=True) + left = None if tips is None else _git_dir_run( + Path(common), "rev-list", "-n1", *tips, "--not", "--branches", "--tags", "--remotes") if tips else "" + if left is None or left.strip(): + return f"{where}: commits only this worktree's reflog holds", [] + else: + gitdirs[os.path.realpath(dotgit)] = dotgit + if (loss := ignored_checkout_loss(root)): + return f"{where}: {loss}", [] + links = _git_dir_run(dotgit, "--work-tree", str(root), "ls-files", "-s") + if links is None: + return f"{where}: unreadable index", [] + for ln in links.splitlines(): + if ln.startswith("160000 ") and (sub := root / ln.split("\t", 1)[1] / ".git").is_dir(): + gitdirs[os.path.realpath(sub)] = sub + for g in list(gitdirs.values()): # submodule stores: modules/, nested ones too + for dirpath, dirnames, _ in os.walk(g / "modules"): + if looks_like_repository(dirpath): + gitdirs[os.path.realpath(dirpath)] = Path(dirpath) + dirnames[:] = ["modules"] if "modules" in dirnames else [] + stamp, bundles = time.strftime("%Y%m%dT%H%M%S"), [] + for key, g in sorted(gitdirs.items()): + if any(t.exists() and not (os.path.realpath(t) + "/").startswith(inside) for t in linked_work_trees(g)): + return f"{key}: a worktree outside this item uses it", bundles + if (why := bundle_unpushed(g, f"{stamp}-{re.sub(r'[^A-Za-z0-9._-]+', '_', key).strip('_')[-120:]}", bundles)): + return why, bundles + return "", bundles + + def submodule_loss(worktree: Path, gitdir: Path, seen: set[str] | None = None, depth: int = 0) -> str: """"" when every submodule repository this checkout would take with it loses nothing. @@ -5088,9 +5271,9 @@ def _survey(families, idle_hours, min_bytes, known, evidence, defer_bulk=False, elif cmux_next_cap_only(fam, path, idle, idle_hours): reasons.append(f"cmux-next-ci is {gib(cmux_next_ci_over()).strip()} over its size cap " "(least recently used first)") - if (fam.mirror_idle_hours is not None and idle >= fam.mirror_idle_hours - and not derived_data(fam, path)): - reasons.append(f"idle {fam.mirror_idle_hours:g}h+: its checkouts are mirrors") + if mirror_due(fam, path, idle): + reasons.append(f"idle {fam.mirror_idle_hours:g}h+: its checkouts are mirrors " + "(unpushed commits go to a verified bundle in ~/archive first)") if jobs is not None and verdict in ("recent", "reclaimable"): reasons.append(reasons_jobs) if policy_note: @@ -6121,6 +6304,8 @@ def apply(items: list[Item], families: dict[str, Family], receipt: Path, released = 0 _NESTED_WALKS.clear() # re-observe every checkout right before deleting every = list(families.values()) if all_families is None else all_families + if any(f.mirror_idle_hours is not None for f in every): + prune_mirror_archive() def family_of(item: Item) -> Family: """The family that listed the item: ids repeat across roots (one interrupted-deletions family per @@ -6287,7 +6472,15 @@ def apply(items: list[Item], families: dict[str, Family], receipt: Path, and (g := family_git_state(fam, p)) != "none" and (g != "git" or family_git_loss(fam, p))): outcome = "changed:git" # the checkout gained something to lose since the survey + elif mirror_due(fam, p, idle_now) and (bundled := archive_mirror(p))[0]: + outcome = "blocked:bundle-failed" # never deleted without a verified bundle of what is unpushed + i.reasons.append(bundled[0]) + RUN.setdefault("bundle_failed", []).append(i.path) + print(f"glaeda-disk: ALERT: kept {i.path}: bundling its unpushed git work failed: {bundled[0]}", + file=sys.stderr) else: + if mirror_due(fam, p, idle_now) and bundled[1]: + i.reasons.append("unpushed git work bundled to " + ", ".join(map(os.fspath, bundled[1]))) whole_state = fam.id == CMUX_NEXT_CI_FAMILY and p in cmux_next_ci_state_roots(fam.root) try: common = git_common_dir(p) if fam.git_disposable else "" @@ -6349,7 +6542,7 @@ def apply(items: list[Item], families: dict[str, Family], receipt: Path, except (OSError, subprocess.SubprocessError) as e: outcome = f"incomplete:{e.__class__.__name__}" is_sweep = halt is not None if receipt_sweep is None else receipt_sweep - if outcome != "reclaimed" and (outcome.startswith("changed:") or outcome.startswith("incomplete:")): + if outcome != "reclaimed" and outcome.startswith(("changed:", "incomplete:", "blocked:")): # Keep the post-apply health/accounting view honest when a candidate became # busy or otherwise failed the final recheck after the survey. i.verdict = "blocked" @@ -8248,7 +8441,8 @@ def write_run_line(started: float, free_before: int | None) -> None: "reached": None if target is None or after is None else after >= target, "blocked": RUN.get("blocked", {}), "duration_s": round(time.time() - started, 1), "skipped": RUN.get("skipped"), "cmux_next_ci_over_gib": RUN.get("cmux_next_ci_over_gib"), - "caller": RUN.get("caller"), "evict_owner": _owner_line(RUN.get("evict_owner"))} + "caller": RUN.get("caller"), "evict_owner": _owner_line(RUN.get("evict_owner")), + "bundle_failed": RUN.get("bundle_failed", [])} with contextlib.suppress(OSError): RUN_LOG.parent.mkdir(parents=True, exist_ok=True) with RUN_LOG.open("a") as f: diff --git a/scripts/test-glaeda-disk.py b/scripts/test-glaeda-disk.py index ed113c97e..68851696e 100644 --- a/scripts/test-glaeda-disk.py +++ b/scripts/test-glaeda-disk.py @@ -1469,12 +1469,12 @@ def test_a_retired_workers_cache_entries_idle_a_week_are_mirrors(self) -> None: fam = gd.hq_cache_family(cache) self.assertEqual(fam.reference_root, self.root) week = gd.CMUX_JOB_MIRROR_HOURS - for name, hours in (("cmux-workspaces", week + 22), ("swiftpm", week + 142), ("fresh-workspaces", 72), + for name, hours in (("cmux-workspaces", week + 22), ("swiftpm", week + 142), ("fresh-workspaces", 6), ("leased", week + 50), ("cmux-tui-target", week + 121)): make(cache / name, age_hours=hours) for name in ("cmux-workspaces", "fresh-workspaces", "leased"): (cache / name / "key/src/.git").mkdir(parents=True) - t = time.time() - (72 if name == "fresh-workspaces" else week + 22) * 3600 + t = time.time() - (6 if name == "fresh-workspaces" else week + 22) * 3600 for d in (cache / name / "key/src/.git", cache / name / "key/src", cache / name / "key", cache / name): os.utime(d, (t, t)) (cache / "swiftpm/repositories/dep/.git").mkdir(parents=True) @@ -1498,9 +1498,10 @@ def git_state(f, path): # a cargo target dir too large to search live = {Path(i.path).name: i.verdict for i in gd.survey([fam], 24, 0)} self.assertEqual({k: i.verdict for k, i in items.items()}, { "cmux-workspaces": "reclaimable", "swiftpm": "reclaimable", "cmux-tui-target": "reclaimable", - "fresh-workspaces": "git-checkout", # idle three days: its clone still has to lose nothing + "fresh-workspaces": "git-checkout", # idle 6 h, under the mirror age: its clone has to lose nothing "leased": "in-use"}) # a running fleet job names it - self.assertIn("idle 168h+: its checkouts are mirrors", items["cmux-workspaces"].reasons) + self.assertIn("idle 12h+: its checkouts are mirrors (unpushed commits go to a verified bundle in ~/archive " + "first)", items["cmux-workspaces"].reasons) self.assertIn("live lease or running job", items["leased"].reasons) self.assertEqual({k for k, v in live.items() if v != "report-only"}, {"leased"}) reclaim = [i for i in items.values() if i.verdict == "reclaimable"] @@ -1513,7 +1514,9 @@ def git_state(f, path): # a cargo target dir too large to search with contextlib.redirect_stdout(io.StringIO()), mock.patch.object(gd, "owner_retired", return_value=True), \ mock.patch.object(gd, "family_git_state", git_state), \ mock.patch.object(gd, "process_evidence", return_value=late), \ - mock.patch.object(gd, "command_lines", return_value=late[1]): + mock.patch.object(gd, "command_lines", return_value=late[1]), \ + mock.patch.object(gd, "HOME", self.root / "home"), \ + mock.patch.object(gd, "archive_mirror", return_value=("", [])): # bundling: the cmux-job tests gd.apply(reclaim, {fam.id: fam}, receipt, None, 24) self.assertEqual(sorted(p.name for p in cache.iterdir()), ["cmux-tui-target", "fresh-workspaces", "leased", "swiftpm"]) @@ -5499,7 +5502,7 @@ def test_linux_families(self) -> None: # scratch clones and leaked files are judged on any filesystem, not only a tmpfs self.assertTrue(tmp.files and tmp.git_disposable) - def test_cmux_job_units_go_but_unpushed_checkouts_stay(self) -> None: + def test_cmux_job_units_go_and_unpushed_checkouts_go_to_a_bundle(self) -> None: job = self.home / ".cache/cmux-job" # as on cmux13s: SwiftPM clones inside DerivedData, and each slot's clone one level below cmux-base (job / "reload-cloud-ios/DerivedData/slot-1/Build").mkdir(parents=True) @@ -5526,8 +5529,8 @@ def test_cmux_job_units_go_but_unpushed_checkouts_stay(self) -> None: gd.process_evidence = saved verdicts = {Path(i.path).relative_to(self.home).as_posix(): (i.family, i.verdict) for i in items} self.assertEqual(verdicts[".cache/cmux-job/reload-cloud-ios/DerivedData"], ("cmux-job-cache", "reclaimable")) - # a commit no remote holds keeps its checkout - self.assertEqual(verdicts[".cache/cmux-job/reload-cloud/cmux-base"], ("cmux-job-cache", "git-checkout")) + # idle past 12 h: a commit no remote holds goes to a bundle, then the checkout goes + self.assertEqual(verdicts[".cache/cmux-job/reload-cloud/cmux-base"], ("cmux-job-cache", "reclaimable")) self.assertNotIn(".cache/cmux-job", verdicts) # not listed again as a report-only tool cache receipt = self.home / "receipt.jsonl" fams_by_id = {f.id: f for f in fams} @@ -5540,7 +5543,8 @@ def test_cmux_job_units_go_but_unpushed_checkouts_stay(self) -> None: gd.process_evidence = saved self.assertFalse((job / "reload-cloud-ios/DerivedData").exists()) self.assertEqual([c.name for c in (job / "reload-cloud-ios").iterdir()], []) # no half-deleted leftover - self.assertTrue((base / ".git").is_dir()) + self.assertFalse(base.exists()) + self.assertEqual(len(list((self.home / "archive/glaeda-disk").glob("*.bundle"))), 1) # a delete interrupted after its rename leaves a tree nothing else will ever clean up dead = subprocess.Popen(["true"]) dead.wait() @@ -5560,7 +5564,7 @@ def test_cmux_job_units_go_but_unpushed_checkouts_stay(self) -> None: gd.process_evidence = saved self.assertEqual(again[str(left)], ("interrupted-deletions", "reclaimable")) - def test_idle_cmux_job_mirrors_go_after_a_week_unless_a_job_holds_them(self) -> None: + def test_idle_cmux_job_mirrors_go_after_12h_bundled_unless_a_job_holds_them(self) -> None: """2026-10-11: 107 GiB of reload-cloud trees idle since Sep 28-Oct 2 on eight minis, all kept as "git-checkout (checkout below the top level)", so glaeda-disk reported 0.0 G reclaimable.""" job = self.home / ".cache/cmux-job" @@ -5587,9 +5591,9 @@ def age(rel: str, hours: float) -> None: os.utime(job / rel, (t, t)) age("", 8 * 24) - age("slots", 3 * 24) # idle, but not a week: its local commit still keeps it + age("slots", 26) # idle past 12 h and the pass window: its local commit goes to a bundle and the tree goes fams = [f for f in gd.default_families() if f.id == "cmux-job-cache"] - self.assertEqual(fams[0].mirror_idle_hours, 168.0) + self.assertEqual(fams[0].mirror_idle_hours, gd.DEFAULT_AGE_HOURS) with mock.patch.object(gd, "process_evidence", return_value=([], f"rsync -a {job}/reload-cloud/work/slot-1/ x\n")): items = {Path(i.path).relative_to(job).as_posix(): i for i in gd.survey(fams, 24, 0)} @@ -5599,9 +5603,10 @@ def age(rel: str, hours: float) -> None: "reload-cloud/work": "in-use", # a running command names a slot in it "reload-cloud-ios/cmux-base": "in-use", # its unit's work//.lock "reload-cloud-ios/work": "in-use", - "slots/host.1": "git-checkout", + "slots/host.1": "reclaimable", "SourcePackages/host.1": "reclaimable"}) - self.assertIn("idle 168h+: its checkouts are mirrors", items["reload-cloud/cmux-base"].reasons) + self.assertIn("idle 12h+: its checkouts are mirrors (unpushed commits go to a verified bundle in ~/archive " + "first)", items["reload-cloud/cmux-base"].reasons) receipt = self.home / "receipt.jsonl" with contextlib.redirect_stdout(io.StringIO()), \ mock.patch.object(gd, "process_evidence", return_value=([], "")): @@ -5609,8 +5614,105 @@ def age(rel: str, hours: float) -> None: self.assertFalse((job / "reload-cloud/cmux-base").exists()) self.assertFalse((job / "SourcePackages/host.1").exists()) self.assertTrue((job / "reload-cloud/work/slot-1/.git").is_dir()) - self.assertTrue((job / "slots/host.1/.git").is_dir()) + self.assertFalse((job / "slots/host.1").exists()) self.assertTrue((job / "reload-cloud-ios/cmux-base/slot-2/.git").is_dir()) + # no remote: each deleted clone's whole history is in a verified bundle + bundles = sorted((self.home / "archive/glaeda-disk").iterdir()) + self.assertEqual([("cmux-base" in b.name, "host.1" in b.name) for b in bundles], [(True, False), (False, True)]) + for b in bundles: + self.assertRegex(b.name, gd.MIRROR_BUNDLE_RE) + self.assertIn("refs/heads/", subprocess.run(["git", "bundle", "list-heads", str(b)], capture_output=True, + text=True, check=True).stdout) + + + def mirror_unit(self, srv_commits: int = 1) -> tuple[Path, Path, dict]: + """A cmux-job reload unit idle a day whose clone's origin is a trusted stand-in for GitHub.""" + env = {**os.environ, "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", + "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"} + srv = self.home / "srv" + subprocess.run(["git", "init", "-q", "-b", "main", str(srv)], check=True, env=env) + for n in range(srv_commits): + subprocess.run(["git", "-C", str(srv), "commit", "-q", "--allow-empty", "-m", f"c{n}"], check=True, env=env) + unit = self.home / ".cache/cmux-job/reload-cloud/cmux-base" + subprocess.run(["git", "clone", "-q", str(srv), str(unit / "slot-1")], check=True, env=env) + saved = gd.TRUSTED_LOCAL_REMOTES + gd.TRUSTED_LOCAL_REMOTES = (str(srv),) + self.addCleanup(setattr, gd, "TRUSTED_LOCAL_REMOTES", saved) + return unit, unit / "slot-1", env + + def apply_mirror(self, unit: Path) -> tuple[dict, str]: + t = time.time() - 24 * 3600 + for dirpath, dirnames, filenames in os.walk(unit): + for n in dirnames + filenames: + os.utime(os.path.join(dirpath, n), (t, t), follow_symlinks=False) + os.utime(unit, (t, t)) + fams = [f for f in gd.default_families() if f.id == "cmux-job-cache"] + items = [i for i in gd.survey(fams, 24, 0) if i.path == os.fspath(unit)] + self.assertEqual([i.verdict for i in items], ["reclaimable"]) + receipt, err = self.home / "receipt.jsonl", io.StringIO() + with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err): + gd.apply(items, {f.id: f for f in fams}, receipt, None, 24) + return json.loads(receipt.read_text().splitlines()[-1]), err.getvalue() + + def bundle_heads(self) -> dict[str, str]: + heads = {} + for b in sorted((self.home / "archive/glaeda-disk").glob("*.bundle")): + subprocess.run(["git", "bundle", "verify", str(b)], cwd=self.home / "srv", check=True, capture_output=True) + out = subprocess.run(["git", "bundle", "list-heads", str(b)], capture_output=True, text=True, check=True) + heads.update({ln.split(" ")[1]: ln.split(" ")[0] for ln in out.stdout.splitlines()}) + return heads + + def test_a_mirrors_unpushed_commit_and_local_refs_survive_in_a_verified_bundle(self) -> None: + unit, clone, env = self.mirror_unit(srv_commits=2) + subprocess.run(["git", "-C", str(clone), "commit", "-q", "--allow-empty", "-m", "unpushed"], check=True, env=env) + subprocess.run(["git", "-C", str(clone), "tag", "only-here", "origin/main~1"], check=True, env=env) + subprocess.run(["git", "-C", str(clone), "branch", "side", "origin/main"], check=True, env=env) + subprocess.run(["git", "-C", str(clone), "checkout", "-q", "--detach", "HEAD"], check=True, env=env) + subprocess.run(["git", "-C", str(clone), "commit", "-q", "--allow-empty", "-m", "detached"], check=True, env=env) + detached = subprocess.run(["git", "-C", str(clone), "rev-parse", "HEAD"], capture_output=True, text=True, + check=True).stdout.strip() + want = {r: subprocess.run(["git", "-C", str(clone), "rev-parse", r], capture_output=True, text=True, + check=True).stdout.strip() for r in ("refs/heads/main", "refs/tags/only-here", + "refs/heads/side")} + rec, _ = self.apply_mirror(unit) + self.assertEqual(rec["outcome"], "reclaimed") + self.assertFalse(unit.exists()) + heads = self.bundle_heads() + self.assertEqual({r: heads.get(r) for r in want}, want) + self.assertEqual(heads.get(gd.ARCHIVE_REF + detached), detached) # a commit only HEAD held + self.assertTrue(any("bundled to" in r for r in rec["reasons"])) + + def test_a_mirror_whose_bundle_fails_is_kept_and_alerts(self) -> None: + unit, clone, env = self.mirror_unit() + subprocess.run(["git", "-C", str(clone), "commit", "-q", "--allow-empty", "-m", "unpushed"], check=True, env=env) + (self.home / "archive").write_text("not a directory") # the bundle cannot be written + gd.RUN.pop("bundle_failed", None) + rec, err = self.apply_mirror(unit) + self.assertEqual((rec["outcome"], rec["verdict"]), ("blocked:bundle-failed", "blocked")) + self.assertIn("ALERT", err) + self.assertEqual(gd.RUN.pop("bundle_failed"), [os.fspath(unit)]) + self.assertTrue((clone / ".git").is_dir()) + self.assertEqual(subprocess.run(["git", "-C", str(clone), "for-each-ref", gd.ARCHIVE_REF], + capture_output=True, text=True).stdout, "") + + def test_a_mirror_with_nothing_unpushed_goes_without_a_bundle(self) -> None: + unit, _, _ = self.mirror_unit() + rec, _ = self.apply_mirror(unit) + self.assertEqual(rec["outcome"], "reclaimed") + self.assertFalse(unit.exists()) + self.assertFalse((self.home / "archive/glaeda-disk").exists()) + + def test_expired_mirror_bundles_are_pruned_and_nothing_else(self) -> None: + archive = self.home / "archive/glaeda-disk" + archive.mkdir(parents=True) + names = ["20260901T000000-a.expires-2026-10-01.bundle", "20261020T000000-b.expires-2026-11-19.bundle", + "20261026T000000-c.expires-2026-10-26.bundle", "a.expires-2026-10-01.bundle", "notes.txt"] + for n in names: + (archive / n).write_text("x") + (self.home / "archive/other.expires-2026-10-01.bundle").write_text("x") # not in its own directory + self.assertEqual(gd.prune_mirror_archive("2026-10-26"), 1) + self.assertEqual(sorted(p.name for p in archive.iterdir()), sorted(names[1:])) + self.assertTrue((self.home / "archive/other.expires-2026-10-01.bundle").exists()) def test_interrupted_deletions_finish_once_their_deleter_is_gone(self) -> None: """cmux8s and cmuxs-mac-mini-3, 2026-10-11: 9.3 and 7.8 GiB renamed aside by a killed glaeda-disk sat From 510e97e8739c0632588b5661d4f4ed1f7bc1fdfa Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 26 Oct 2026 14:56:00 -0700 Subject: [PATCH 09/28] glaeda-disk: a reserved host keeps the 168 h mirror age Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- scripts/glaeda-disk | 2 +- scripts/test-glaeda-disk.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/glaeda-disk b/scripts/glaeda-disk index 1d842843f..8995e3332 100755 --- a/scripts/glaeda-disk +++ b/scripts/glaeda-disk @@ -5508,7 +5508,7 @@ LEGACY_AGES = {"DEFAULT_AGE_HOURS": 24.0, "KEPT_BUILD_IDLE_HOURS": 24.0, "IDLE_S "PARKED_NO_LOG_IDLE_HOURS": 168.0, "CMUX_NEXT_CI_MIN_IDLE_HOURS": 72.0, "CANONICAL_ADMISSION_IDLE_HOURS": 24.0, "IDLE_SEED_HOURS": 12.0, "RUNTIME_IDLE_DAYS": 14.0, "CHECKOUT_BUILD_IDLE_HOURS": 24.0, "TMP_TEST_RESIDUE_IDLE_HOURS": 48.0, - "RUNNER_STALE_CHECKOUT_HOURS": float("inf")} + "RUNNER_STALE_CHECKOUT_HOURS": float("inf"), "CMUX_JOB_MIRROR_HOURS": 168.0} LEGACY_POLICY = {"soft_idle_hours": 24.0, "stale_marker_max_hours": 48.0, "cmux_next_ci_host_idle_hours": 48.0} diff --git a/scripts/test-glaeda-disk.py b/scripts/test-glaeda-disk.py index 68851696e..af410e1a4 100644 --- a/scripts/test-glaeda-disk.py +++ b/scripts/test-glaeda-disk.py @@ -6421,6 +6421,7 @@ def test_a_reserved_host_keeps_the_ages_before_twelve_hours(self) -> None: self.assertEqual(gd.POLICY["stale_marker_max_hours"], 5.0, "a disk-policy.json value stays") self.assertEqual((gd.DEFAULT_AGE_HOURS, gd.CMUX_NEXT_CI_MIN_IDLE_HOURS, gd.RUNTIME_IDLE_DAYS), (24.0, 72.0, 14.0)) self.assertEqual(gd.RUNNER_STALE_CHECKOUT_HOURS, float("inf"), "no above-floor checkout eviction") + self.assertEqual(gd.CMUX_JOB_MIRROR_HOURS, 168.0, "no 12 h mirror deletion on a reserved host") def test_the_shipped_list_parses_and_matches_the_registry_where_it_is_checked_out(self) -> None: shipped = json.loads((ROOT / "config/fleet-reservations.json").read_text()) From 592273516e415a887ba4aef699cab7b741b33c3f Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 26 Oct 2026 15:14:01 -0700 Subject: [PATCH 10/28] glaeda-disk: read the fleet reservation list only from the host; fail closed on any malformed entry The list no longer ships in this repository. glaeda-disk and glaeda-fleet-cas-prune read it only from ~/.local/share/glaeda/fleet-reservations.json in the home of the user that owns /Users/Shared/cmux-build-fleet (written there by the fleet's own writer), plus FLEET_ROOT/reservation.json. A missing, unreadable or malformed list (hosts not a list of names, a bad time on any entry, aliases not lists) reads as reserved, whatever the host-local file says. Also from review: the prune registers the glaeda-disk module in sys.modules before running it, so its dataclasses load and the gate is real; the reservation gate runs before store_gc; --runner-caches is held to the same floor-only rule; and a reserved host's pressure starts at the floor itself, not floor + low_margin. Test fixtures use made-up ids, hosts and owners. Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- config/fleet-reservations.json | 24 --------- scripts/glaeda-disk | 73 +++++++++++++++++-------- scripts/glaeda-fleet-cas-prune | 33 ++++++++---- scripts/glaeda-mini-setup | 10 +--- scripts/test-glaeda-disk.py | 75 ++++++++++++++++++++------ scripts/test-glaeda-fleet-cas-prune.py | 22 ++++++++ scripts/test-glaeda-mini-setup.py | 2 +- 7 files changed, 158 insertions(+), 81 deletions(-) delete mode 100644 config/fleet-reservations.json diff --git a/config/fleet-reservations.json b/config/fleet-reservations.json deleted file mode 100644 index 8ad1c4b32..000000000 --- a/config/fleet-reservations.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "schema": "glaeda-fleet-reservations/v1", - "note": "A copy of the `reservations` list (id, owner, hosts, from, until) in leo-workspace machines/fleet-registry.json (tools/fleet_reservations.py reads the same entries), with each reserved host's registry aliases. glaeda-disk and glaeda-fleet-cas-prune read it from ~/.local/share/glaeda/fleet-reservations.json: on a host an active entry names, only floor-pressure eviction runs, at the pre-2026-10-26 ages. An entry lapses by itself after `until`. Update it with the registry: scripts/test-glaeda-disk.py compares the two where the registry is checked out.", - "reservations": [ - { - "id": "lawrence-2026-10-25", - "owner": "lawrencecchen", - "hosts": [ - "cmux16", - "cmux17", - "cmux18", - "cmux19", - "cmux20", - "cmux21", - "cmux22", - "cmux23", - "cmux24" - ], - "from": "2026-10-25T22:00:00Z", - "until": "2026-10-28T23:59:59Z" - } - ], - "aliases": {} -} diff --git a/scripts/glaeda-disk b/scripts/glaeda-disk index c3a261c00..ee873a8ac 100755 --- a/scripts/glaeda-disk +++ b/scripts/glaeda-disk @@ -5212,17 +5212,30 @@ def runner_floor_gib(home: Path = HOME) -> float: return floor -# A fleet reservation (leo-workspace machines/fleet-registry.json `reservations`, shipped as -# config/fleet-reservations.json and installed here by glaeda-mini-setup) hands a host to one person for a while. On a -# reserved fleet mini only floor-pressure eviction runs: no idle sweep, and a pressure run acts only below the runner -# floor and works only to the floor plus target_margin_gib, so state its holder keeps idle is not aged out. The -# host-local fleet reservation (FLEET_ROOT/reservation.json, glaeda-reservation/v1, the one the runner hook reads) -# counts too. Fails closed: an unreadable or missing list on a fleet mini reads as reserved. -FLEET_RESERVATIONS = HOME / ".local/share/glaeda/fleet-reservations.json" +# A fleet reservation hands a host to one person for a while. On a reserved fleet mini only floor-pressure eviction +# runs: no idle sweep, and a pressure run acts only below the runner floor and works only back to the floor plus +# target_margin_gib, so state its holder keeps idle is not aged out. The list is never part of this repository: the +# fleet's own writer puts it on each host, in the home of the user that owns FLEET_ROOT +# (~/.local/share/glaeda/fleet-reservations.json, schema glaeda-fleet-reservations/v1: +# {"reservations": [{"id", "owner", "hosts": [...], "from", "until"}], "aliases": {host: [...]}}). The host-local +# reservation (FLEET_ROOT/reservation.json, glaeda-reservation/v1, the one the runner hook reads) counts too. Fails +# closed: a missing, unreadable or malformed list reads as reserved, whatever the host-local file says. +FLEET_RESERVATIONS: Path | None = None # None: fleet_reservations_path(); tests point it at a fixture +FLEET_RESERVATIONS_NAME = ".local/share/glaeda/fleet-reservations.json" UPDATE_CONFIG = HOME / ".config/glaeda/update.json" HOST_RESERVATION_LOCAL = FLEET_ROOT / "reservation.json" +def fleet_reservations_path() -> Path: + """The list in the home of the user that owns FLEET_ROOT (the fleet account, also when this runs as root); + this user's home when FLEET_ROOT or its owner cannot be read.""" + if FLEET_RESERVATIONS is not None: + return FLEET_RESERVATIONS + with contextlib.suppress(OSError, KeyError): + return Path(pwd.getpwuid(FLEET_ROOT.stat().st_uid).pw_dir) / FLEET_RESERVATIONS_NAME + return HOME / FLEET_RESERVATIONS_NAME + + def reservation_time(value: Any) -> float: """An ISO 8601 time (Z or an offset; a naive one is UTC) as Unix seconds; ValueError otherwise.""" if not isinstance(value, str) or not value: @@ -5245,25 +5258,40 @@ def host_names() -> set[str]: return {n for n in names if n} +def _str_list(value: Any, what: str) -> list[str]: + if not isinstance(value, list) or not all(isinstance(v, str) and v for v in value): + raise ValueError(f"{what} must be a list of names") + return value + + def host_reservation(now: float | None = None, path: Path | None = None, names: set[str] | None = None, local: Path | None = None) -> str: """Why this host is reserved now ("" when it is not): an active fleet reservation naming it (or an alias), an - active host-local reservation, or a reservation list that cannot be read.""" + active host-local reservation, or a reservation list that is missing, unreadable or malformed anywhere.""" now = time.time() if now is None else now names = host_names() if names is None else {n.lower() for n in names} try: - doc = json.loads((FLEET_RESERVATIONS if path is None else path).read_text()) - entries, aliases = doc["reservations"], doc.get("aliases") or {} + doc = json.loads((fleet_reservations_path() if path is None else path).read_text()) + if not isinstance(doc, dict): + raise ValueError("malformed") + entries, aliases = doc["reservations"], doc.get("aliases", {}) if not isinstance(entries, list) or not isinstance(aliases, dict): raise ValueError("malformed") - for e in entries: - hosts = {str(h).lower() for h in e["hosts"]} - hosts |= {str(a).lower() for h in e["hosts"] for a in aliases.get(h, [])} - if names & hosts and reservation_time(e["from"]) <= now <= reservation_time(e["until"]): - return f"reserved for {e.get('owner', '?')} until {e['until']} ({e.get('id', '?')})" + alias = {str(h).lower(): {a.lower() for a in _str_list(v, "aliases")} for h, v in aliases.items()} + hit = "" + for e in entries: # every entry is checked, so one malformed entry anywhere fails closed + if not isinstance(e, dict): + raise ValueError("malformed entry") + hosts = {h.lower() for h in _str_list(e["hosts"], "hosts")} + hosts |= {a for h in list(hosts) for a in alias.get(h, ())} + start, end = reservation_time(e["from"]), reservation_time(e["until"]) + if not hit and names & hosts and start <= now <= end: + hit = f"reserved for {e.get('owner', '?')} until {e['until']} ({e.get('id', '?')})" + if hit: + return hit except (OSError, ValueError, KeyError, TypeError) as error: return f"fleet reservations unreadable ({type(error).__name__}); kept as reserved" - with contextlib.suppress(OSError, ValueError, TypeError, AttributeError): + with contextlib.suppress(OSError, ValueError, TypeError, AttributeError, KeyError): held = json.loads((HOST_RESERVATION_LOCAL if local is None else local).read_text()) if held.get("schema") == "glaeda-reservation/v1" and int(held["since"]) <= now < int(held["until"]): return f"host reservation by {held.get('owner', '?')} until {held['until']}" @@ -5271,7 +5299,8 @@ def host_reservation(now: float | None = None, path: Path | None = None, names: def filesystems(fams: list[Family], low: str, target: str, - tmpfs_low: str = "75%", tmpfs_target: str = "90%", floor_gib: float = 0.0) -> dict[int, Fs]: + tmpfs_low: str = "75%", tmpfs_target: str = "90%", floor_gib: float = 0.0, + reserved: bool = False) -> dict[int, Fs]: out: dict[int, Fs] = {} for f in fams: dev = f.root.stat().st_dev @@ -5281,9 +5310,11 @@ def filesystems(fams: list[Family], low: str, target: str, lo, tg = (tmpfs_low, tmpfs_target) if tmpfs else (low, target) lo_b, tg_b = space(lo, total), space(tg, total) # a runner's disk floor (runner_floor_gib) lifts both on HOME's volume, where the gate measures it, - # never past half the disk, as space() keeps a small disk from permanent pressure + # never past half the disk, as space() keeps a small disk from permanent pressure; a reserved host's + # pressure starts at the floor itself, not the floor plus low_margin_gib if floor_gib > 0 and not tmpfs and dev == HOME.stat().st_dev: - lo_b = max(lo_b, min(int((floor_gib + POLICY["low_margin_gib"]) * GIB), total // 2)) + lo_margin = 0.0 if reserved else POLICY["low_margin_gib"] + lo_b = max(lo_b, min(int((floor_gib + lo_margin) * GIB), total // 2)) tg_b = max(tg_b, min(int((floor_gib + POLICY["target_margin_gib"]) * GIB), total // 2)) out[dev] = Fs(dev, str(mount_point(f.root)), free, total, lo_b, tg_b, tmpfs) return out @@ -7763,7 +7794,7 @@ def _main_impl(argv: list[str] | None = None) -> int: a.target = f"{a.need + POLICY['target_margin_gib']:g}" # before any family is built: a reserved host's families keep its holder's ages (keep_legacy_ages, once 12 h lands) - reserved = host_reservation() if role == "fleet-mini" and DARWIN and a.apply and not a.runner_caches else "" + reserved = host_reservation() if role == "fleet-mini" and DARWIN and a.apply else "" global RUNNER_CACHE_PROFILE RUNNER_CACHE_PROFILE = bool(a.runner_caches) fams = runner_cache_families() if RUNNER_CACHE_PROFILE else default_families(role) @@ -7820,7 +7851,7 @@ def _main_impl(argv: list[str] | None = None) -> int: a.low, a.target = f"{floor_gib:g}", f"{floor_gib + POLICY['target_margin_gib']:g}" print(f"{reserved}: only floor-pressure eviction runs here") fss = filesystems(fams or [Family("home", HOME, False, "")], a.low, a.target, - a.tmpfs_low, a.tmpfs_target, floor_gib) + a.tmpfs_low, a.tmpfs_target, floor_gib, reserved=bool(reserved)) space(a.emergency, GIB) except ValueError: ap.error(f"--low/--target/--tmpfs-*/--emergency take GiB or N%: {a.low!r}, {a.target!r}, " diff --git a/scripts/glaeda-fleet-cas-prune b/scripts/glaeda-fleet-cas-prune index ee7e55161..38eddcc14 100755 --- a/scripts/glaeda-fleet-cas-prune +++ b/scripts/glaeda-fleet-cas-prune @@ -62,14 +62,27 @@ GC_EVERY = 86400 # seconds between fleet-store gc runs GC_REPORT = re.compile(r"kept entries naming no stored object: (\d+)") -def reserved_above_floor() -> str: - """Why this host is left alone: a fleet reservation holds it (glaeda-disk's host_reservation, read from the - glaeda-disk installed beside this script) and free disk is at or above its runner floor. "" when not.""" +def load_glaeda_disk(path: Path | None = None): + """glaeda-disk installed beside this script, as a module (registered in sys.modules, which its dataclasses need).""" + path = Path(__file__).with_name("glaeda-disk") if path is None else path + name = "glaeda_disk_for_prune" + loader = importlib.machinery.SourceFileLoader(name, os.fspath(path)) + spec = importlib.util.spec_from_loader(name, loader) + gd = importlib.util.module_from_spec(spec) + sys.modules[name] = gd try: - loader = importlib.machinery.SourceFileLoader("glaeda_disk_for_prune", os.fspath(Path(__file__).with_name("glaeda-disk"))) - spec = importlib.util.spec_from_loader(loader.name, loader) - gd = importlib.util.module_from_spec(spec) loader.exec_module(gd) + except BaseException: + sys.modules.pop(name, None) + raise + return gd + + +def reserved_above_floor(gd=None) -> str: + """Why this host is left alone: a fleet reservation holds it (glaeda-disk's host_reservation, which fails closed) + and free disk is at or above its runner floor. "" when not.""" + try: + gd = load_glaeda_disk() if gd is None else gd why = gd.host_reservation() floor = gd.runner_floor_gib() except Exception as error: # noqa: BLE001 (fail closed: no glaeda-disk to ask, so treat the host as reserved) @@ -218,6 +231,10 @@ def main(argv: list[str] | None = None) -> int: help="on the store host: fleet-cas gc keeps entries used within this many days") a = ap.parse_args(argv) rec: dict = {"at": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "apply": a.apply} + if a.apply and (held := reserved_above_floor()): # a reserved host's caches are its holder's until the floor + rec["result"] = f"deferred: {held}" + print(json.dumps(rec)) + return 0 if a.apply: rec.update(store_gc(ROOT, a.gc_keep_days)) if not (ROOT / "fleet-cas.env").exists(): @@ -226,10 +243,6 @@ def main(argv: list[str] | None = None) -> int: return 0 writer = is_writer(ROOT) rec["role"] = "writer" if writer else "reader" - if a.apply and (held := reserved_above_floor()): # a reserved host's caches are its holder's until the floor - rec["result"] = f"deferred: {held}" - print(json.dumps(rec)) - return 0 commands = running_commands() if commands is None: rec["result"] = "deferred: ps failed, cannot rule out a running build" diff --git a/scripts/glaeda-mini-setup b/scripts/glaeda-mini-setup index dff6d5d09..2d472bc86 100755 --- a/scripts/glaeda-mini-setup +++ b/scripts/glaeda-mini-setup @@ -194,7 +194,6 @@ DAEMON_UNITS = frozenset({"glaeda-gh.service"}) SKILL_SOURCE = REPO / "skills/github-ci-wait/SKILL.md" DAEMON_POLICY_SOURCE = REPO / "config/macos-daemons.json" # desired state of macOS system daemons CLAUDE_ROUTES_SOURCE = REPO / "config/claude-routes.json" # the fleet routes glaeda-mini-health keeps Claude on -FLEET_RESERVATIONS_SOURCE = REPO / "config/fleet-reservations.json" # who holds which fleet host; glaeda-disk reads it # glaeda-update runs this command from a release; it sets this so setup never unloads the job running it UPDATE_ENV = "GLAEDA_UPDATE_RUNNING" OTA_RINGS = ("canary", "stable") @@ -988,11 +987,6 @@ def skill_path(ctx: Context) -> Path: return ctx.home / ".claude/skills/github-ci-wait/SKILL.md" -def fleet_reservations_path(ctx: Context) -> Path: - """config/fleet-reservations.json, where glaeda-disk and glaeda-fleet-cas-prune read it (FLEET_RESERVATIONS).""" - return ctx.home / ".local/share/glaeda/fleet-reservations.json" - - def daemon_policy_path(ctx: Context) -> Path: """config/macos-daemons.json, where glaeda-mini-health and glaeda-daemon-heal read it.""" return ctx.home / ".local/share/glaeda/macos-daemons.json" @@ -1292,8 +1286,7 @@ def plan_install(ctx: Context) -> list[dict[str, Any]]: def plan_hygiene_tools(ctx: Context) -> list[dict[str, Any]]: """The macOS daemon policy and glaeda-worktree-reclaim: the parts of a hygiene install beyond the scripts.""" actions: list[dict[str, Any]] = [] - for dest, source in ((daemon_policy_path(ctx), DAEMON_POLICY_SOURCE), (claude_routes_path(ctx), CLAUDE_ROUTES_SOURCE), - (fleet_reservations_path(ctx), FLEET_RESERVATIONS_SOURCE)): + for dest, source in ((daemon_policy_path(ctx), DAEMON_POLICY_SOURCE), (claude_routes_path(ctx), CLAUDE_ROUTES_SOURCE)): act = file_action(ctx, "tool", dest, source.read_bytes(), 0o644) act["source"] = os.fspath(source) actions.append(act) @@ -1490,7 +1483,6 @@ def plan_uninstall(ctx: Context) -> list[dict[str, Any]]: actions.append({"kind": "tool", "path": os.fspath(skill_path(ctx))}) actions.append({"kind": "tool", "path": os.fspath(daemon_policy_path(ctx))}) actions.append({"kind": "tool", "path": os.fspath(claude_routes_path(ctx))}) - actions.append({"kind": "tool", "path": os.fspath(fleet_reservations_path(ctx))}) for marker in scan_exclusion_markers(ctx): actions.append({"kind": "marker", "path": os.fspath(marker)}) for act in actions: diff --git a/scripts/test-glaeda-disk.py b/scripts/test-glaeda-disk.py index fdc50559c..248819cdd 100644 --- a/scripts/test-glaeda-disk.py +++ b/scripts/test-glaeda-disk.py @@ -22,6 +22,7 @@ import sys import tempfile import time +import types import unittest from dataclasses import replace from unittest import mock @@ -6198,7 +6199,7 @@ def fss(*args: object) -> dict: self.assertEqual(seen, [("54", "70", 30.0), ("55", "71", 50.0)]) class HostReservationTest(unittest.TestCase): - """A reserved fleet mini gets floor-pressure eviction only, at the pre-2026-10-26 ages (config/fleet-reservations.json).""" + """A reserved fleet mini gets floor-pressure eviction only. The list lives only on the host; fixtures are made up.""" def setUp(self) -> None: self.tmp = tempfile.TemporaryDirectory() @@ -6237,25 +6238,63 @@ def test_the_host_local_reservation_counts(self) -> None: "until": now - 1})) self.assertEqual(self.reserved({"cmux31"}), "") - def test_the_shipped_list_parses_and_matches_the_registry_where_it_is_checked_out(self) -> None: - shipped = json.loads((ROOT / "config/fleet-reservations.json").read_text()) - for e in shipped["reservations"]: - gd.reservation_time(e["from"]), gd.reservation_time(e["until"]) - self.assertTrue(e["hosts"] and e["owner"] and e["id"]) - registry = Path(os.environ.get("GLAEDA_FLEET_REGISTRY") - or Path.home() / "Projects/leo-workspace/machines/fleet-registry.json") - if not registry.is_file(): - self.skipTest("leo-workspace's fleet registry is not checked out here") - want = [{k: e[k] for k in ("id", "owner", "hosts", "from", "until") if k in e} - for e in json.loads(registry.read_text()).get("reservations", [])] - self.assertEqual(shipped["reservations"], want, "config/fleet-reservations.json is behind the registry") + def test_a_malformed_list_anywhere_fails_closed(self) -> None: + good = {"id": "r2", "owner": "someone-else", "hosts": ["cmux40"], "from": "2026-10-25T00:00:00Z", + "until": "2026-10-28T00:00:00Z"} + bad_docs = ( + [], # not an object + {"aliases": {}}, # no reservations + {"reservations": {"r1": good}}, + {"reservations": [dict(good, hosts="cmux31")]}, # a string, not a list: never matched letter by letter + {"reservations": [dict(good, hosts=[31])]}, + {"reservations": ["cmux31"]}, + {"reservations": [dict(good, until="soon")]}, # a bad time on an entry naming another host + {"reservations": [good], "aliases": {"cmux40": "cmux-forty"}}, + {"reservations": [good], "aliases": []}, + ) + for doc in bad_docs: + with self.subTest(doc=doc): + self.list.write_text(json.dumps(doc)) + self.assertIn("unreadable", self.reserved({"cmux31"})) + self.list.write_text(json.dumps({"reservations": [good]})) + self.assertEqual(self.reserved({"cmux31"}), "", "a well-formed list without this host") + + def test_a_missing_list_is_reserved_whatever_the_host_local_file_says(self) -> None: + now = int(self.during) + self.local.write_text(json.dumps({"schema": "glaeda-reservation/v1", "owner": "fleet", "since": now - 600, + "until": now - 60})) + self.list.unlink() + self.assertIn("kept as reserved", self.reserved({"cmux31"})) + + def test_the_list_is_read_from_the_fleet_owners_home(self) -> None: + owner = types.SimpleNamespace(pw_dir=os.fspath(self.root / "fleet-home")) + with mock.patch.object(gd, "FLEET_RESERVATIONS", None), mock.patch.object(gd, "FLEET_ROOT", self.root), \ + mock.patch.object(gd.pwd, "getpwuid", return_value=owner) as getpwuid: + self.assertEqual(gd.fleet_reservations_path(), + self.root / "fleet-home/.local/share/glaeda/fleet-reservations.json") + getpwuid.assert_called_once_with(self.root.stat().st_uid) + self.assertIn("kept as reserved", gd.host_reservation(self.during, None, {"cmux31"}, self.local), + "nothing written there yet") + with mock.patch.object(gd, "FLEET_RESERVATIONS", None), \ + mock.patch.object(gd, "FLEET_ROOT", self.root / "absent"): + self.assertEqual(gd.fleet_reservations_path(), gd.HOME / gd.FLEET_RESERVATIONS_NAME) + + def test_a_reserved_hosts_pressure_starts_at_the_floor_itself(self) -> None: + fam = gd.Family("home", gd.HOME, False, "") + total = 1000 * gd.GIB + with mock.patch.object(gd, "free_bytes", return_value=(100 * gd.GIB, total)): + plain = next(iter(gd.filesystems([fam], "1", "1", floor_gib=30.0).values())) + held = next(iter(gd.filesystems([fam], "30", "50", floor_gib=30.0, reserved=True).values())) + self.assertEqual(plain.low, int((30 + gd.POLICY["low_margin_gib"]) * gd.GIB)) + self.assertEqual(held.low, 30 * gd.GIB) + self.assertEqual(held.target, int((30 + gd.POLICY["target_margin_gib"]) * gd.GIB)) def run_disk(self, *args: str, reserved: str, floor: float, free_gib: float): """glaeda-disk's main on a fake fleet mini; returns (exit code, output, the families' low/target it used).""" seen, run = {}, {} - def filesystems(fams, low, target, *rest): - seen.update(low=low, target=target) + def filesystems(fams, low, target, *rest, reserved=False): + seen.update(low=low, target=target, **({"reserved": True} if reserved else {})) run.update(gd.RUN) raise SystemExit(0) # stop before any survey: the gate is what is under test with mock.patch.object(gd, "host_reservation", return_value=reserved), \ @@ -6283,8 +6322,12 @@ def test_a_reserved_host_runs_only_floor_pressure_to_the_floor_plus_the_margin(s self.assertEqual(seen, {}) code, out, seen, run = self.run_disk("--pressure", "--idle", reserved=why, floor=30.0, free_gib=50) margin = gd.POLICY["target_margin_gib"] - self.assertEqual(seen, {"low": "30", "target": f"{30 + margin:g}"}) + self.assertEqual(seen, {"low": "30", "target": f"{30 + margin:g}", "reserved": True}) self.assertEqual(run["reserved"], why) + # the runner-cache profile is held to the same rule + code, out, seen, run = self.run_disk("--runner-caches", reserved=why, floor=30.0, free_gib=50) + self.assertIn("nothing deleted", out) + self.assertEqual(seen, {}) # an unreserved host keeps its policy code, out, seen, run = self.run_disk("--pressure", reserved="", floor=30.0, free_gib=50) self.assertEqual(seen, {"low": gd.POLICY["low"], "target": gd.POLICY["target"]}) diff --git a/scripts/test-glaeda-fleet-cas-prune.py b/scripts/test-glaeda-fleet-cas-prune.py index ae0886fe4..31b82b9b5 100644 --- a/scripts/test-glaeda-fleet-cas-prune.py +++ b/scripts/test-glaeda-fleet-cas-prune.py @@ -205,6 +205,28 @@ def test_reserved_above_floor_reads_glaeda_disk_and_fails_closed(self) -> None: real = self.saved[3] with mock.patch.object(fp.importlib.machinery, "SourceFileLoader", side_effect=OSError("gone")): self.assertIn("kept as reserved", real()) + # the real glaeda-disk loads (its dataclasses need the module in sys.modules) and answers from a made-up list + gd = fp.load_glaeda_disk(ROOT / "scripts/glaeda-disk") + listing = Path(self.tmp.name) / "fleet-reservations.json" + listing.write_text(json.dumps({"reservations": [{"id": "r9", "owner": "someone", "hosts": ["cmux30"], + "from": "2000-01-01T00:00:00Z", + "until": "2999-01-01T00:00:00Z"}]})) + gd.FLEET_RESERVATIONS = listing + gd.host_names = lambda: {"cmux30"} + gd.runner_floor_gib = lambda: 0.0 + self.assertIn("reserved for someone", real(gd)) + gd.host_names = lambda: {"cmux31"} + gd.HOST_RESERVATION_LOCAL = Path(self.tmp.name) / "no-reservation.json" + self.assertEqual(real(gd), "") + listing.unlink() + self.assertIn("kept as reserved", real(gd)) + + def test_a_reserved_host_never_runs_store_gc(self) -> None: + fp.reserved_above_floor = lambda: "reserved for someone until 2999-01-01T00:00:00Z (r9); 80 GiB free, floor 30" + with mock.patch.object(fp, "store_gc", side_effect=AssertionError("gc ran on a reserved host")): + rec = self.run_main("--apply") + self.assertTrue(rec["result"].startswith("deferred: reserved"), rec) + if __name__ == "__main__": unittest.main() diff --git a/scripts/test-glaeda-mini-setup.py b/scripts/test-glaeda-mini-setup.py index da1f4c0eb..2686c5f99 100644 --- a/scripts/test-glaeda-mini-setup.py +++ b/scripts/test-glaeda-mini-setup.py @@ -713,7 +713,7 @@ def test_hygiene_only_installs_tools_and_agents_on_any_mac(self) -> None: self.assertTrue(os.access(self.home / ".local/bin" / name, os.X_OK), name) for label in ("disk-pressure", "disk-dedupe", "worktree-reclaim"): self.assertTrue((self.home / f"Library/LaunchAgents/com.teamleaderleo.glaeda.{label}.plist").is_file()) - for name in ("macos-daemons.json", "claude-routes.json", "fleet-reservations.json"): # read off the repo + for name in ("macos-daemons.json", "claude-routes.json"): # what glaeda-mini-health reads off the repo self.assertEqual((self.home / ".local/share/glaeda" / name).read_bytes(), (ROOT / "config" / name).read_bytes(), name) # the build-host profile still blocks on the same machine From 66e2f58bb8f36f14beed70ffd4b2c6d0424b0ca7 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 26 Oct 2026 15:20:58 -0700 Subject: [PATCH 11/28] glaeda-disk: mirror bundles trust only the server, never overwrite, keep dirty trees, recheck before delete Addresses the Codex review of #1625: 1. "held" comes only from what the servers confirm (ls-remote heads and tags, and fetch --negotiate-only acks, asked one commit at a time when a batch fails); stale remote-tracking refs are only candidates. 2. Bundle names carry a hash of the repository's real path and ref, the pid and the time, and are linked into place with os.link, so an existing bundle is never replaced. 3. A work tree with tracked edits, untracked non-ignored files or files hidden from status keeps the item (blocked:uncommitted); ignored build output still goes. 4. Checkouts inside ignored directories are bundled like the others. 5. An unreadable submodule store keeps the item (os.walk onerror). 6. Right before the delete, fresh process evidence and a snapshot of every ref, HEAD, reflog and work tree status taken before bundling are checked again; any change keeps the tree. 7. Pruning deletes only bundles manifest.jsonl lists, only at the sha256 it recorded, and only after the process-evidence gate. Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- scripts/glaeda-disk | 311 ++++++++++++++++++++++++++---------- scripts/test-glaeda-disk.py | 129 +++++++++++++-- 2 files changed, 343 insertions(+), 97 deletions(-) diff --git a/scripts/glaeda-disk b/scripts/glaeda-disk index 8995e3332..07ec7d3d6 100755 --- a/scripts/glaeda-disk +++ b/scripts/glaeda-disk @@ -164,9 +164,12 @@ build output or copies. Nested or unverifiable checkouts stay protected. A mirror family's item (cmux-job-cache, the hq build cache: mirror_idle_hours) idle DEFAULT_AGE_HOURS goes whole, but first archive_mirror writes every commit no remote holds, and every branch or tag only this host -has, into git bundles under ~/archive/glaeda-disk (named -.expires-.bundle, pruned by that -name after MIRROR_ARCHIVE_DAYS) and checks each with `git bundle verify`. A bundle that fails keeps the tree: -the receipt says blocked:bundle-failed, the run line lists it, and stderr alerts. +has (what the servers confirm, never a local remote-tracking ref), into git bundles under ~/archive/glaeda-disk, +each checked with `git bundle verify`, never written over another, and listed with its sha256 in manifest.jsonl; +only listed bundles at that hash are pruned, MIRROR_ARCHIVE_DAYS later. A bundle that fails keeps the tree: +the receipt says blocked:bundle-failed, the run line lists it, and stderr alerts. Uncommitted or untracked work, +which no bundle holds, keeps it too (blocked:uncommitted); ignored build output goes. Right before the delete, +fresh process evidence and the repositories' refs as at bundle time are checked again. """ from __future__ import annotations @@ -4386,34 +4389,73 @@ def repo_loss(gitdir: Path) -> str: return "" -# archive_mirror's bundles: only files this tool named so are ever pruned, only in its own directory +# archive_mirror's bundles: only files its manifest lists, at the sha256 recorded there, are ever pruned MIRROR_ARCHIVE_DAYS = 30 -MIRROR_BUNDLE_RE = re.compile(r"\d{8}T\d{6}-[A-Za-z0-9._-]+\.expires-(\d{4}-\d{2}-\d{2})\.bundle") +MIRROR_BUNDLE_RE = re.compile(r"\d{8}T\d{6}-\d+-[0-9a-f]{16}-[A-Za-z0-9._-]*\.expires-(\d{4}-\d{2}-\d{2})\.bundle") ARCHIVE_REF = "refs/glaeda-archive/" # a commit only a reflog or HEAD holds is named for the bundle meanwhile +NEGOTIATE_MAX = 1024 # more candidate commits than this are not asked about: their history is bundled instead def mirror_archive() -> Path: return HOME / "archive" / "glaeda-disk" +def _sha256(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + def prune_mirror_archive(today: str | None = None) -> int: - """Delete this tool's bundles whose name's expiry date has passed; nothing else in ~/archive.""" + """Delete the bundles manifest.jsonl lists whose expiry has passed, each only while it is a regular file + with the sha256 the manifest recorded for it; nothing the manifest does not list, nothing else in ~/archive.""" today = today or time.strftime("%Y-%m-%d") - root, gone = mirror_archive(), 0 - if root.is_symlink() or not root.is_dir(): + root, gone, kept = mirror_archive(), 0, [] + manifest = root / "manifest.jsonl" + if root.is_symlink() or not root.is_dir() or manifest.is_symlink() or not manifest.is_file(): return 0 - for e in _entries(root): - m = MIRROR_BUNDLE_RE.fullmatch(e.name) - if m and m.group(1) < today and e.is_file(follow_symlinks=False): - with contextlib.suppress(OSError): - os.unlink(e.path) - gone += 1 + try: + lines = manifest.read_text().splitlines() + except OSError: + return 0 + for line in lines: + try: + e = json.loads(line) + name, sha, expires = str(e["name"]), str(e["sha256"]), str(e["expires"]) + except (ValueError, KeyError, TypeError): + kept.append(line) + continue + f = root / name + if not MIRROR_BUNDLE_RE.fullmatch(name) or expires >= today: + kept.append(line) + elif f.is_file() and not f.is_symlink(): + try: + if _sha256(f) == sha: + f.unlink() + gone += 1 + continue + except OSError: + pass + kept.append(line) # changed or unreadable: never deleted, still listed + with contextlib.suppress(OSError): + tmp = manifest.with_name(f".manifest.{os.getpid()}.tmp") + tmp.write_text("".join(ln + "\n" for ln in kept)) + os.replace(tmp, manifest) return gone +def _bundle_name(key: str, ref: str = "") -> str: + """Unique per repository (a hash of its real path), per ref, per pass and per process; readable tail.""" + digest = hashlib.sha256(f"{key}\0{ref}".encode()).hexdigest()[:16] + tail = re.sub(r"[^A-Za-z0-9._-]+", "_", f"{key}{'-' + ref if ref else ''}").strip("_")[-80:] + return f"{time.strftime('%Y%m%dT%H%M%S')}-{os.getpid()}-{digest}-{tail}" + + def _write_bundle(gitdir: Path, name: str, revs: list[str], heads: set[str], written: list[Path]) -> str: - """`git bundle create` from rev-list arguments, checked by `git bundle verify` and its heads, then moved - to its archive name (appended to written). "" or why it failed.""" + """`git bundle create` from rev-list arguments, checked by `git bundle verify` and its heads, then linked + to its archive name (never over an existing file) and recorded in the manifest. "" or why it failed.""" expires = time.strftime("%Y-%m-%d", time.localtime(time.time() + MIRROR_ARCHIVE_DAYS * 86400)) dest = mirror_archive() / f"{name}.expires-{expires}.bundle" tmp = dest.with_name(f".{dest.name}.{os.getpid()}.tmp") @@ -4425,8 +4467,10 @@ def _write_bundle(gitdir: Path, name: str, revs: list[str], heads: set[str], wri listed = _git_dir_run(gitdir, "bundle", "list-heads", str(tmp)) if listed is None or not heads <= {ln.split(" ", 1)[-1] for ln in listed.splitlines()}: return f"bundle of {gitdir} lacks a ref" - os.replace(tmp, dest) + os.link(tmp, dest) # FileExistsError rather than replacing another bundle written.append(dest) + with (dest.parent / "manifest.jsonl").open("a") as m: + m.write(json.dumps({"name": dest.name, "sha256": _sha256(dest), "expires": expires}) + "\n") return "" except OSError as e: return f"bundle of {gitdir}: {e.__class__.__name__}" @@ -4435,10 +4479,27 @@ def _write_bundle(gitdir: Path, name: str, revs: list[str], heads: set[str], wri tmp.unlink(missing_ok=True) -def bundle_unpushed(gitdir: Path, stem: str, written: list[Path]) -> str: - """Bundle what deleting this repository loses: commits no remote holds (found as repo_loss finds them, - with the heads and tags the remotes advertise counted as held) and every branch or tag only it has. - "" or why it failed; the bundles go to written.""" +def _server_acks(gitdir: Path, remotes: list[str], commits: list[str]) -> set[str]: + """The commits among `commits` some remote confirms it holds (`fetch --negotiate-only`, as remote_has). + A server that decides it is ready answers with a packfile and git gives up on the batch, so a failed batch + is asked one commit at a time; a commit no answer confirms is not held (its history is bundled).""" + def ask(remote: str, batch: list[str]) -> set[str] | None: + out = _git_dir_run(gitdir, "-c", "protocol.version=2", "fetch", "--negotiate-only", + *[f"--negotiation-tip={c}" for c in batch], remote, network=True) + return None if out is None else set(out.split()) & set(batch) + acked: set[str] = set() + for r in remotes: + for k in range(0, len(commits), 64): + batch = [c for c in commits[k:k + 64] if c not in acked] + got = ask(r, batch) if batch else set() + acked |= got if got is not None else {c for c in batch if ask(r, [c])} + return acked + + +def bundle_unpushed(gitdir: Path, key: str, written: list[Path]) -> str: + """Bundle what deleting this repository loses: commits no remote server confirms (what ls-remote advertises + and what `fetch --negotiate-only` acknowledges; local remote-tracking refs are only candidates to ask + about), and every branch or tag the servers do not have at that commit. "" or why it failed.""" tips = local_tips(gitdir) for wt in [p for p in (gitdir / "worktrees").glob("*") if p.is_dir()] if tips is not None else []: own = local_tips(wt, worktree_only=True) @@ -4448,54 +4509,52 @@ def bundle_unpushed(gitdir: Path, stem: str, written: list[Path]) -> str: if tips is None or remotes is None or refs is None: return f"unreadable refs or remotes in {gitdir}" local: dict[str, tuple[str, str]] = {} # refname -> (object, commit) - held: dict[str, set[str]] = {} # refname (as the remote names it) -> objects a remote holds there + tracking: set[str] = set() for line in refs.splitlines(): name, obj, peeled = (line.split("\t") + ["", ""])[:3] - r = next((r for r in remotes if name.startswith(f"refs/remotes/{r}/")), None) - if r is not None: - held.setdefault("refs/heads/" + name[len(f"refs/remotes/{r}/"):], set()).add(obj) - elif not name.startswith("refs/remotes/"): + if name.startswith("refs/remotes/"): + tracking.add(peeled or obj) + else: local[name] = (obj, peeled or obj) - answered = not remotes + held: dict[str, set[str]] = {} # refname on a server -> objects it advertises there for r in remotes: - if (out := _git_dir_run(gitdir, "ls-remote", "--heads", "--tags", r, network=True, timeout=120)) is not None: - answered = True - for ln in out.splitlines(): - obj, _, name = ln.partition("\t") - held.setdefault(name.removesuffix("^{}"), set()).add(obj) - if not answered: - return f"no remote of {gitdir} answered" - objs = sorted({o for s in held.values() for o in s}) - have = _git_dir_run(gitdir, "cat-file", "--batch-check=%(objectname)", stdin="\n".join(objs) + "\n") \ - if objs else "" + if (out := _git_dir_run(gitdir, "ls-remote", "--heads", "--tags", r, network=True, timeout=120)) is None: + return f"remote {r} of {gitdir} did not answer" + for ln in out.splitlines(): + obj, _, name = ln.partition("\t") + held.setdefault(name.removesuffix("^{}"), set()).add(obj) + candidates = sorted({*tips, *tracking, *(o for s in held.values() for o in s)}) + have = _git_dir_run(gitdir, "cat-file", "--batch-check=%(objectname) %(objecttype)", + stdin="\n".join(candidates) + "\n") if candidates else "" if have is None: return f"unreadable objects in {gitdir}" - known = [ln for ln in have.splitlines() if not ln.endswith(" missing")] + present = {ln.split()[0] for ln in have.splitlines() if ln.endswith(" commit") or ln.endswith(" tag")} + advertised = {o for s in held.values() for o in s} & present + ask = sorted((present & ({*tips} | tracking)) - advertised) + acked = _server_acks(gitdir, remotes, ask) if len(ask) <= NEGOTIATE_MAX else set() + known = sorted(advertised | (acked & present)) out = _git_dir_run(gitdir, "rev-list", "--stdin", stdin="\n".join([*tips, "--not", *known]) + "\n", timeout=600) if out is None: return f"unreadable history in {gitdir}" unpushed = set(out.split()) - loose = [t for t in tips if t in unpushed] - if loose and len(loose) <= 64 and remote_has(gitdir, loose): - unpushed, loose = set(), [] - # the refs to keep: every local one on an unpushed commit, and every branch or tag the remotes lack + # the refs to keep: every local one on an unpushed commit, and every branch or tag the servers lack there keep = {n for n, (_, c) in local.items() if c in unpushed} only_here = sorted(n for n, (o, _) in local.items() if n.startswith(("refs/heads/", "refs/tags/")) and n not in keep and o not in held.get(n, ())) - loose = sorted(set(loose) - {local[n][1] for n in keep}) + loose = sorted({t for t in tips if t in unpushed} - {local[n][1] for n in keep}) if loose and _git_dir_run(gitdir, "update-ref", "--stdin", stdin="".join( f"create {ARCHIVE_REF}{t} {t}\n" for t in loose)) is None: return f"cannot name the loose commits in {gitdir}" try: heads = keep | {ARCHIVE_REF + t for t in loose} - if heads: - if (why := _write_bundle(gitdir, stem, [*sorted(heads), "--not", *known], heads, written)): - return why - for n in only_here: # its commit is on a remote: the bundle holds that commit and the name + if heads and (why := _write_bundle(gitdir, _bundle_name(key), [*sorted(heads), "--not", *known], + heads, written)): + return why + for n in only_here: # its commit is on a server: the bundle holds that commit and the name parents = _git_dir_run(gitdir, "rev-parse", f"{local[n][1]}^@") if parents is None or (why := _write_bundle( - gitdir, f"{stem}-{re.sub(r'[^A-Za-z0-9._-]+', '_', n)}", [n, "--not", *parents.split()], {n}, written)): + gitdir, _bundle_name(key, n), [n, "--not", *parents.split()], {n}, written)): return why or f"unreadable {n} in {gitdir}" return "" finally: @@ -4504,26 +4563,87 @@ def bundle_unpushed(gitdir: Path, stem: str, written: list[Path]) -> str: f"delete {ARCHIVE_REF}{t}\n" for t in loose)) -def archive_mirror(path: Path) -> tuple[str, list[Path]]: +def _ignored_checkouts(root: Path) -> list[Path] | None: + """Checkouts inside a checkout's ignored directories (vendor/ in .gitignore), as ignored_checkout_loss + finds them; build-output directories are skipped as there. None when they cannot all be found.""" + try: + r = subprocess.run(["git", "-C", str(root), "status", "--porcelain", "-z", "--ignored=matching", + "--untracked-files=normal"], capture_output=True, text=True, timeout=60, + env={**os.environ, "GIT_OPTIONAL_LOCKS": "0"}) + except (OSError, subprocess.TimeoutExpired): + return None + if r.returncode != 0: + return None + found: list[Path] = [] + for entry in r.stdout.split("\0"): + if entry.startswith("!! ") and entry.endswith("/") and Path(entry[3:-1]).name not in REPRODUCIBLE_DIRS: + if (inner := checkouts(root / entry[3:-1], NESTED_DEPTH, NESTED_BUDGET)) is None: + return None + found += inner + return found + + +def _work_tree_dirty(root: Path) -> str: + """Tracked edits, untracked files that are not ignored, or files hidden from status: a bundle holds none.""" + status = _git_dir_run(root / ".git", "--work-tree", str(root), "status", "--porcelain", + "--untracked-files=all", "--ignore-submodules=none") + flags = _git_dir_run(root / ".git", "--work-tree", str(root), "ls-files", "-v") + if status is None or flags is None: + return "unreadable work tree" + if status.strip() or any(ln[:1].islower() or ln[:1] == "S" for ln in flags.splitlines()): + return "uncommitted or untracked changes" + return "" + + +def mirror_git_snapshot(gitdirs: list[Path], trees: list[Path]) -> list[str | None]: + """Every ref, HEAD and reflog size of the item's repositories and the status of its work trees: compared + before and after archive_mirror's bundles, so a commit made meanwhile keeps the tree.""" + snap: list[str | None] = [] + for g in gitdirs: + snap.append(_git_dir_run(g, "for-each-ref", "--format=%(refname) %(objectname)")) + for d in [g, *(p for p in (g / "worktrees").glob("*") if p.is_dir())]: + snap.append(_git_dir_run(d, "rev-parse", "HEAD")) + logs = d / "logs" + try: + snap.append(" ".join(f"{f}:{f.stat().st_size}" for f in sorted(logs.rglob("*")) if f.is_file()) + if logs.is_dir() else "") + except OSError: + snap.append(None) # unreadable: never matches, so the tree stays + snap += [_git_dir_run(t / ".git", "--work-tree", str(t), "status", "--porcelain", "--untracked-files=all") + for t in trees] + return snap + + +def archive_mirror(path: Path) -> tuple[str, list[Path], tuple[list[Path], list[Path], list[str | None]]]: """Before a mirror item goes: bundle_unpushed for every repository it takes along, found as _nested_loss - finds them (checkouts, bare repositories, a linked worktree's repository inside it, submodule stores and - embedded submodules). A tree whose repositories cannot all be found or bundled is kept. - ("" or why it must stay, the bundles written).""" - roots = nested_checkouts(path) - if roots is None: - return "too deep or large to find every checkout", [] - inside = os.path.realpath(path) + "/" + finds them (checkouts, bare repositories, a linked worktree's repository inside it, checkouts in ignored + directories, submodule stores and embedded submodules). A tree with uncommitted work, or whose repositories + cannot all be found or bundled, is kept. ("" or why it must stay, the bundles written, what to recheck).""" + none: tuple[list[Path], list[Path], list[str | None]] = ([], [], []) + queue = nested_checkouts(path) + if queue is None: + return "too deep or large to find every checkout", [], none + queue, inside = list(queue), os.path.realpath(path) + "/" gitdirs: dict[str, Path] = {} - for root in roots: + trees: list[Path] = [] + while queue: + root = queue.pop() where, dotgit = root.relative_to(path), root / ".git" if not os.path.lexists(dotgit): gitdirs[os.path.realpath(root)] = root - elif dotgit.is_symlink(): - return f"{where}: .git is a symlink", [] - elif dotgit.is_file(): # a linked worktree: bundle its repository when that goes too + continue + if dotgit.is_symlink(): + return f"{where}: .git is a symlink", [], none + if (dirty := _work_tree_dirty(root)): + return f"uncommitted: {where}: {dirty}", [], none + trees.append(root) + if (inner := _ignored_checkouts(root)) is None: + return f"{where}: too deep or large to find every checkout in its ignored files", [], none + queue += inner + if dotgit.is_file(): # a linked worktree: bundle its repository when that goes too common, own = git_common_dir(root), _git_dir_run(dotgit, "rev-parse", "--absolute-git-dir") if not common or own is None: - return f"{where}: unreadable worktree", [] + return f"{where}: unreadable worktree", [], none if (os.path.realpath(common) + "/").startswith(inside): gitdirs[os.path.realpath(common)] = Path(common) continue @@ -4531,29 +4651,46 @@ def archive_mirror(path: Path) -> tuple[str, list[Path]]: left = None if tips is None else _git_dir_run( Path(common), "rev-list", "-n1", *tips, "--not", "--branches", "--tags", "--remotes") if tips else "" if left is None or left.strip(): - return f"{where}: commits only this worktree's reflog holds", [] - else: - gitdirs[os.path.realpath(dotgit)] = dotgit - if (loss := ignored_checkout_loss(root)): - return f"{where}: {loss}", [] - links = _git_dir_run(dotgit, "--work-tree", str(root), "ls-files", "-s") - if links is None: - return f"{where}: unreadable index", [] - for ln in links.splitlines(): - if ln.startswith("160000 ") and (sub := root / ln.split("\t", 1)[1] / ".git").is_dir(): - gitdirs[os.path.realpath(sub)] = sub + return f"{where}: commits only this worktree's reflog holds", [], none + continue + gitdirs[os.path.realpath(dotgit)] = dotgit + links = _git_dir_run(dotgit, "--work-tree", str(root), "ls-files", "-s") + if links is None: + return f"{where}: unreadable index", [], none + for ln in links.splitlines(): + if ln.startswith("160000 ") and (sub := root / ln.split("\t", 1)[1] / ".git").is_dir(): + gitdirs[os.path.realpath(sub)] = sub + unreadable: list[str] = [] for g in list(gitdirs.values()): # submodule stores: modules/, nested ones too - for dirpath, dirnames, _ in os.walk(g / "modules"): + for dirpath, dirnames, _ in os.walk(g / "modules", onerror=lambda e: unreadable.append(str(e.filename))): if looks_like_repository(dirpath): gitdirs[os.path.realpath(dirpath)] = Path(dirpath) dirnames[:] = ["modules"] if "modules" in dirnames else [] - stamp, bundles = time.strftime("%Y%m%dT%H%M%S"), [] + if [u for u in unreadable if os.path.lexists(u)]: # a missing modules/ is no submodule store + return f"unreadable submodule store {unreadable[0]}", [], none + snap = mirror_git_snapshot(list(gitdirs.values()), trees) + bundles: list[Path] = [] for key, g in sorted(gitdirs.items()): if any(t.exists() and not (os.path.realpath(t) + "/").startswith(inside) for t in linked_work_trees(g)): - return f"{key}: a worktree outside this item uses it", bundles - if (why := bundle_unpushed(g, f"{stamp}-{re.sub(r'[^A-Za-z0-9._-]+', '_', key).strip('_')[-120:]}", bundles)): - return why, bundles - return "", bundles + return f"{key}: a worktree outside this item uses it", bundles, none + if (why := bundle_unpushed(g, key, bundles)): + return why, bundles, none + return "", bundles, (list(gitdirs.values()), trees, snap) + + +def mirror_moved(fam: Family, path: Path, recheck: tuple[list[Path], list[Path], list[str | None]]) -> str: + """Right before a bundled mirror goes: fresh process evidence, and its repositories as at bundle time.""" + try: + opened, cmds = process_evidence() + except NoEvidence as e: + return f"no process evidence: {e}" + if in_cwd(os.fspath(path), open_index(opened)) or named_by(os.fspath(path), cmds) or holds_socket(path) \ + or job_busy(fam, path): + return "in use while its git work was bundled" + gitdirs, trees, snap = recheck + if None in snap or mirror_git_snapshot(gitdirs, trees) != snap: + return "its git refs or work trees changed while they were bundled" + return "" def submodule_loss(worktree: Path, gitdir: Path, seen: set[str] | None = None, @@ -6304,8 +6441,6 @@ def apply(items: list[Item], families: dict[str, Family], receipt: Path, released = 0 _NESTED_WALKS.clear() # re-observe every checkout right before deleting every = list(families.values()) if all_families is None else all_families - if any(f.mirror_idle_hours is not None for f in every): - prune_mirror_archive() def family_of(item: Item) -> Family: """The family that listed the item: ids repeat across roots (one interrupted-deletions family per @@ -6319,6 +6454,8 @@ def apply(items: list[Item], families: dict[str, Family], receipt: Path, print(f"refusing to delete anything: {e}", file=sys.stderr) return 0 cwds = open_index(cwds) + if any(f.mirror_idle_hours is not None for f in every): + prune_mirror_archive() lsof_at, lsof_items = time.monotonic(), 0 archived = archived_seeds() if any(i.family == "cmux-seeds" for i in items) else frozenset() # Among pressure hot-tier items, evict greatest reuse distance, then oldest use. @@ -6473,11 +6610,17 @@ def apply(items: list[Item], families: dict[str, Family], receipt: Path, and (g != "git" or family_git_loss(fam, p))): outcome = "changed:git" # the checkout gained something to lose since the survey elif mirror_due(fam, p, idle_now) and (bundled := archive_mirror(p))[0]: - outcome = "blocked:bundle-failed" # never deleted without a verified bundle of what is unpushed + # never deleted without a verified bundle of what is unpushed; uncommitted work no bundle holds + dirty = bundled[0].startswith("uncommitted: ") + outcome = "blocked:uncommitted" if dirty else "blocked:bundle-failed" i.reasons.append(bundled[0]) - RUN.setdefault("bundle_failed", []).append(i.path) - print(f"glaeda-disk: ALERT: kept {i.path}: bundling its unpushed git work failed: {bundled[0]}", - file=sys.stderr) + if not dirty: + RUN.setdefault("bundle_failed", []).append(i.path) + print(f"glaeda-disk: ALERT: kept {i.path}: bundling its unpushed git work failed: {bundled[0]}", + file=sys.stderr) + elif mirror_due(fam, p, idle_now) and (moved := mirror_moved(fam, p, bundled[2])): + outcome = "changed:in-use" # its bundles stay in the archive; the tree stays too + i.reasons.append(moved) else: if mirror_due(fam, p, idle_now) and bundled[1]: i.reasons.append("unpushed git work bundled to " + ", ".join(map(os.fspath, bundled[1]))) diff --git a/scripts/test-glaeda-disk.py b/scripts/test-glaeda-disk.py index af410e1a4..05ebf1df7 100644 --- a/scripts/test-glaeda-disk.py +++ b/scripts/test-glaeda-disk.py @@ -1516,7 +1516,7 @@ def git_state(f, path): # a cargo target dir too large to search mock.patch.object(gd, "process_evidence", return_value=late), \ mock.patch.object(gd, "command_lines", return_value=late[1]), \ mock.patch.object(gd, "HOME", self.root / "home"), \ - mock.patch.object(gd, "archive_mirror", return_value=("", [])): # bundling: the cmux-job tests + mock.patch.object(gd, "archive_mirror", return_value=("", [], ([], [], []))): # bundling: the cmux-job tests gd.apply(reclaim, {fam.id: fam}, receipt, None, 24) self.assertEqual(sorted(p.name for p in cache.iterdir()), ["cmux-tui-target", "fresh-workspaces", "leased", "swiftpm"]) @@ -5617,8 +5617,9 @@ def age(rel: str, hours: float) -> None: self.assertFalse((job / "slots/host.1").exists()) self.assertTrue((job / "reload-cloud-ios/cmux-base/slot-2/.git").is_dir()) # no remote: each deleted clone's whole history is in a verified bundle - bundles = sorted((self.home / "archive/glaeda-disk").iterdir()) - self.assertEqual([("cmux-base" in b.name, "host.1" in b.name) for b in bundles], [(True, False), (False, True)]) + bundles = sorted((self.home / "archive/glaeda-disk").glob("*.bundle")) + self.assertEqual(sorted(("cmux-base" in b.name, "host.1" in b.name) for b in bundles), + [(False, True), (True, False)]) for b in bundles: self.assertRegex(b.name, gd.MIRROR_BUNDLE_RE) self.assertIn("refs/heads/", subprocess.run(["git", "bundle", "list-heads", str(b)], capture_output=True, @@ -5640,7 +5641,7 @@ def mirror_unit(self, srv_commits: int = 1) -> tuple[Path, Path, dict]: self.addCleanup(setattr, gd, "TRUSTED_LOCAL_REMOTES", saved) return unit, unit / "slot-1", env - def apply_mirror(self, unit: Path) -> tuple[dict, str]: + def apply_mirror(self, unit: Path, between=None) -> tuple[dict, str]: t = time.time() - 24 * 3600 for dirpath, dirnames, filenames in os.walk(unit): for n in dirnames + filenames: @@ -5649,6 +5650,8 @@ def apply_mirror(self, unit: Path) -> tuple[dict, str]: fams = [f for f in gd.default_families() if f.id == "cmux-job-cache"] items = [i for i in gd.survey(fams, 24, 0) if i.path == os.fspath(unit)] self.assertEqual([i.verdict for i in items], ["reclaimable"]) + if between is not None: + between() receipt, err = self.home / "receipt.jsonl", io.StringIO() with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err): gd.apply(items, {f.id: f for f in fams}, receipt, None, 24) @@ -5675,7 +5678,7 @@ def test_a_mirrors_unpushed_commit_and_local_refs_survive_in_a_verified_bundle(s check=True).stdout.strip() for r in ("refs/heads/main", "refs/tags/only-here", "refs/heads/side")} rec, _ = self.apply_mirror(unit) - self.assertEqual(rec["outcome"], "reclaimed") + self.assertEqual(rec["outcome"], "reclaimed", rec["reasons"]) self.assertFalse(unit.exists()) heads = self.bundle_heads() self.assertEqual({r: heads.get(r) for r in want}, want) @@ -5702,17 +5705,117 @@ def test_a_mirror_with_nothing_unpushed_goes_without_a_bundle(self) -> None: self.assertFalse(unit.exists()) self.assertFalse((self.home / "archive/glaeda-disk").exists()) - def test_expired_mirror_bundles_are_pruned_and_nothing_else(self) -> None: + def git(self, repo: Path, *args: str, env: dict | None = None) -> str: + return subprocess.run(["git", "-C", str(repo), *args], capture_output=True, text=True, check=True, + env=env).stdout.strip() + + def test_a_branch_the_server_deleted_is_bundled_despite_its_stale_tracking_ref(self) -> None: + unit, clone, env = self.mirror_unit() + srv = self.home / "srv" + self.git(srv, "checkout", "-q", "-b", "feat", env=env) + self.git(srv, "commit", "-q", "--allow-empty", "-m", "feat", env=env) + self.git(srv, "checkout", "-q", "main", env=env) + self.git(clone, "fetch", "-q", "origin", env=env) + self.git(clone, "branch", "mine", "origin/feat", env=env) + feat = self.git(clone, "rev-parse", "origin/feat") + self.git(srv, "branch", "-q", "-D", "feat", env=env) # origin/feat in the clone is stale now + self.git(srv, "reflog", "expire", "--expire=now", "--all", env=env) + self.git(srv, "gc", "-q", "--prune=now", env=env) + rec, _ = self.apply_mirror(unit) + self.assertEqual(rec["outcome"], "reclaimed") + self.assertEqual(self.bundle_heads().get("refs/heads/mine"), feat) + + def test_bundle_names_never_collide_or_overwrite(self) -> None: + self.assertNotEqual(gd._bundle_name("/a/b_c")[16:], gd._bundle_name("/a/b/c")[16:]) + self.assertNotEqual(gd._bundle_name("/r", "refs/heads/a_b")[16:], gd._bundle_name("/r", "refs/heads/a/b")[16:]) + _, clone, env = self.mirror_unit() + self.git(clone, "commit", "-q", "--allow-empty", "-m", "unpushed", env=env) + written: list[Path] = [] + args = (clone / ".git", "same", ["refs/heads/main", "--not", "origin/main"], {"refs/heads/main"}, written) + self.assertEqual(gd._write_bundle(*args), "") + first = written[0].read_bytes() + self.git(clone, "commit", "-q", "--allow-empty", "-m", "another", env=env) + self.assertIn("FileExistsError", gd._write_bundle(*args)) + self.assertEqual((written, written[0].read_bytes()), ([written[0]], first)) + + def test_a_mirror_with_uncommitted_work_is_kept_but_ignored_output_goes(self) -> None: + unit, clone, _ = self.mirror_unit() + with (clone / ".git/info/exclude").open("a") as f: + f.write("build/\n") + (clone / "build").mkdir() + (clone / "build/out.o").write_text("x") + (clone / "notes.txt").write_text("edited on a laptop") + gd.RUN.pop("bundle_failed", None) + rec, err = self.apply_mirror(unit) + self.assertEqual((rec["outcome"], rec["verdict"]), ("blocked:uncommitted", "blocked")) + self.assertTrue((clone / "notes.txt").exists()) + self.assertNotIn("bundle_failed", gd.RUN) + (clone / "notes.txt").unlink() + rec, _ = self.apply_mirror(unit) + self.assertEqual(rec["outcome"], "reclaimed") + + def test_a_checkout_in_ignored_files_has_its_local_refs_bundled(self) -> None: + unit, clone, env = self.mirror_unit() + with (clone / ".git/info/exclude").open("a") as f: + f.write("vendor/\n") + dep = clone / "vendor/dep" + subprocess.run(["git", "init", "-q", "-b", "main", str(dep)], check=True, env=env) + self.git(dep, "commit", "-q", "--allow-empty", "-m", "only here", env=env) + head = self.git(dep, "rev-parse", "HEAD") + rec, _ = self.apply_mirror(unit) + self.assertEqual(rec["outcome"], "reclaimed") + self.assertEqual(self.bundle_heads().get("refs/heads/main"), head) + + def test_an_unreadable_submodule_store_keeps_the_mirror(self) -> None: + unit, clone, _ = self.mirror_unit() + store = clone / ".git/modules/sub" + store.mkdir(parents=True) + try: + rec, _ = self.apply_mirror(unit, between=lambda: os.chmod(store, 0)) + finally: + os.chmod(store, 0o755) + self.assertEqual(rec["outcome"], "blocked:bundle-failed") + self.assertIn("unreadable submodule store", " ".join(rec["reasons"])) + self.assertTrue(clone.is_dir()) + + def test_a_commit_made_while_bundling_keeps_the_mirror(self) -> None: + unit, clone, env = self.mirror_unit() + self.git(clone, "commit", "-q", "--allow-empty", "-m", "unpushed", env=env) + real = gd.archive_mirror + + def racing(path): + out = real(path) + self.git(clone, "commit", "-q", "--allow-empty", "-m", "made meanwhile", env=env) + return out + with mock.patch.object(gd, "archive_mirror", racing): + rec, _ = self.apply_mirror(unit) + self.assertEqual(rec["outcome"], "changed:in-use") + self.assertIn("its git refs or work trees changed while they were bundled", rec["reasons"]) + self.assertTrue(clone.is_dir()) + + def test_only_manifest_listed_bundles_at_their_hash_are_pruned_after_the_evidence_gate(self) -> None: archive = self.home / "archive/glaeda-disk" archive.mkdir(parents=True) - names = ["20260901T000000-a.expires-2026-10-01.bundle", "20261020T000000-b.expires-2026-11-19.bundle", - "20261026T000000-c.expires-2026-10-26.bundle", "a.expires-2026-10-01.bundle", "notes.txt"] - for n in names: - (archive / n).write_text("x") - (self.home / "archive/other.expires-2026-10-01.bundle").write_text("x") # not in its own directory + name = "20260901T000000-1-{:016x}-r.expires-{}.bundle".format + listed = {name(1, "2026-10-01"): "2026-10-01", name(2, "2026-11-19"): "2026-11-19", + name(3, "2026-10-02"): "2026-10-02"} # 3: changed since it was written + for n in [*listed, name(4, "2026-10-01")]: # 4: expired by its name, but not in the manifest + (archive / n).write_text(n) + with (archive / "manifest.jsonl").open("w") as m: + for n, exp in listed.items(): + m.write(json.dumps({"name": n, "sha256": gd._sha256(archive / n), "expires": exp}) + "\n") + (archive / name(3, "2026-10-02")).write_text("changed") + (self.home / "archive/other.bundle").write_text("x") + fams = [f for f in gd.default_families() if f.id == "cmux-job-cache"] + with contextlib.redirect_stderr(io.StringIO()), mock.patch.object( + gd, "process_evidence", side_effect=gd.NoEvidence("ps failed")): + gd.apply([], {f.id: f for f in fams}, self.home / "r.jsonl", None, 24) + self.assertTrue((archive / name(1, "2026-10-01")).exists()) # no evidence: nothing is deleted self.assertEqual(gd.prune_mirror_archive("2026-10-26"), 1) - self.assertEqual(sorted(p.name for p in archive.iterdir()), sorted(names[1:])) - self.assertTrue((self.home / "archive/other.expires-2026-10-01.bundle").exists()) + self.assertEqual(sorted(p.name for p in archive.glob("*.bundle")), + sorted([name(2, "2026-11-19"), name(3, "2026-10-02"), name(4, "2026-10-01")])) + self.assertNotIn(name(1, "2026-10-01"), (archive / "manifest.jsonl").read_text()) + self.assertTrue((self.home / "archive/other.bundle").exists()) def test_interrupted_deletions_finish_once_their_deleter_is_gone(self) -> None: """cmux8s and cmuxs-mac-mini-3, 2026-10-11: 9.3 and 7.8 GiB renamed aside by a killed glaeda-disk sat From c0c4e5fd0a26530b1220a0e054941fba6ce5d5d4 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 26 Oct 2026 15:28:28 -0700 Subject: [PATCH 12/28] glaeda-disk: a stale runner checkout keeps git's operation markers and every worktree's HEAD; explicit policy keys stay Review findings on #1622: - the above-floor veto now also keeps a checkout with FETCH_HEAD.lock, MERGE_HEAD, REBASE_HEAD, a rebase or sequencer dir, CHERRY_PICK_HEAD, REVERT_HEAD or BISECT_LOG (files or dirs, linked worktrees' git dirs too); - a commit that only a linked worktree's HEAD names keeps the checkout; - keep_legacy_ages leaves a key disk-policy.json sets alone even when its value equals the new default (policy_overrides gives the file's keys). Tracked edits and reflog-only commits still go with a runner checkout: actions/checkout's next job runs git clean -ffdx and git reset --hard there, so they never carry from one job to the next. Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- scripts/glaeda-disk | 71 ++++++++++++++++++++++++------------- scripts/test-glaeda-disk.py | 40 +++++++++++++++++++-- 2 files changed, 84 insertions(+), 27 deletions(-) diff --git a/scripts/glaeda-disk b/scripts/glaeda-disk index 53b10e76d..2540d3948 100755 --- a/scripts/glaeda-disk +++ b/scripts/glaeda-disk @@ -1178,29 +1178,34 @@ DEVBOX_TRANSCRIPT_DAYS = 30 DEVBOX_TRANSCRIPT_DELETE_DAYS = 180 -def disk_policy(path: Path | None = None) -> dict[str, Any]: - """POLICY_DEFAULTS overlaid with DISK_POLICY; an unreadable file or a wrongly typed key keeps the default.""" - policy = dict(POLICY_DEFAULTS) +def policy_overrides(path: Path | None = None) -> dict[str, Any]: + """The keys DISK_POLICY sets, each with a value of its default's type; an unreadable file sets none.""" + out: dict[str, Any] = {} try: doc = json.loads((DISK_POLICY if path is None else path).read_text()) except (OSError, ValueError): - return policy + return out if isinstance(doc, dict): for key, default in POLICY_DEFAULTS.items(): value = doc.get(key) if isinstance(default, bool): if isinstance(value, bool): - policy[key] = value + out[key] = value elif isinstance(default, list): if isinstance(value, list) and all(isinstance(v, str) and v for v in value): - policy[key] = list(value) + out[key] = list(value) elif isinstance(default, str): if isinstance(value, str) and value: - policy[key] = value + out[key] = value elif isinstance(value, (int, float)) and not isinstance(value, bool) and value >= 0: if math.isfinite(value): - policy[key] = type(default)(value) - return policy + out[key] = type(default)(value) + return out + + +def disk_policy(path: Path | None = None) -> dict[str, Any]: + """POLICY_DEFAULTS overlaid with DISK_POLICY; an unreadable file or a wrongly typed key keeps the default.""" + return {**POLICY_DEFAULTS, **policy_overrides(path)} POLICY = disk_policy() @@ -1940,32 +1945,46 @@ def runner_checkout_last_use(runner: Path, checkout: Path) -> float: return max(times) -GIT_LOCK_NAMES = ("index.lock", "HEAD.lock", "shallow.lock", "packed-refs.lock", "config.lock", "ORIG_HEAD.lock") +GIT_LOCK_NAMES = ("index.lock", "HEAD.lock", "shallow.lock", "packed-refs.lock", "config.lock", "ORIG_HEAD.lock", + "FETCH_HEAD.lock", "MERGE_HEAD", "REBASE_HEAD", "rebase-merge", "rebase-apply", "CHERRY_PICK_HEAD", + "REVERT_HEAD", "BISECT_LOG", "sequencer") # the home-clone set (HOME_CLONE_GIT_LOCKS) and more def runner_checkout_git_veto(checkout: Path) -> str: - """Why a stale runner checkout must stay for git's sake, or "": git holds a lock in it, or a commit there (HEAD, a - local branch or tag, the stash, any worktree's HEAD) is on no remote-tracking ref. Untracked and ignored files are - a job's build output and go with it. Any doubt (git fails or times out) keeps it.""" + """Why a stale runner checkout must stay for git's sake, or "": git holds a lock or has an operation under way in + it or any of its worktrees (a lock file, MERGE_HEAD, a rebase or sequencer dir), or a commit there (HEAD, every + worktree's HEAD, a local branch or tag, the stash) is on no remote-tracking ref. Any doubt (git fails or times + out) keeps it. + + What goes with it: untracked and ignored files (a job's build output), tracked edits and commits only a reflog + names. This is actions/checkout's own directory: the next job there runs `git clean -ffdx` and `git reset --hard` + and moves HEAD, so a runner checkout never carries those from one job to the next; deleting it loses only what + the runner would throw away itself.""" dotgit = checkout / ".git" if dotgit.is_symlink() or not dotgit.is_dir(): return "not a plain checkout (.git is not a directory)" - with contextlib.suppress(OSError): + try: for root, dirs, files in os.walk(dotgit): + rel = Path(root).relative_to(dotgit).parts dirs[:] = [d for d in dirs if d not in ("objects", "lfs", "logs")] - if any(f in GIT_LOCK_NAMES or (f.endswith(".lock") and "refs" in Path(root).relative_to(dotgit).parts) - for f in files): + if any(n in GIT_LOCK_NAMES for n in (*files, *dirs)) or any( + f.endswith(".lock") and "refs" in rel for f in files): return "git holds a lock in it" + except OSError: + return "git could not say whether it holds a lock" env = {**os.environ, "GIT_OPTIONAL_LOCKS": "0"} + + def git(*args: str, timeout: int = 60) -> str: + return subprocess.run(["git", "-C", os.fspath(checkout), *args], capture_output=True, text=True, + timeout=timeout, env=env, check=True).stdout try: - tips = subprocess.run(["git", "-C", os.fspath(checkout), "for-each-ref", "--format=%(objectname)", - "refs/heads", "refs/tags", "refs/stash"], capture_output=True, text=True, timeout=60, - env=env, check=True).stdout.split() - local = subprocess.run(["git", "-C", os.fspath(checkout), "rev-list", "--max-count=1", "HEAD", *tips, - "--not", "--remotes"], capture_output=True, text=True, timeout=120, env=env, check=True) - except (OSError, subprocess.SubprocessError): + tips = git("for-each-ref", "--format=%(objectname)", "refs/heads", "refs/tags", "refs/stash").split() + heads = [ln.split(" ", 1)[1] for ln in git("worktree", "list", "--porcelain").splitlines() + if ln.startswith("HEAD ")] + local = git("rev-list", "--max-count=1", "HEAD", *heads, *tips, "--not", "--remotes", timeout=120) + except (OSError, subprocess.SubprocessError, IndexError): return "git could not say whether a commit is only here" - return "a commit here is on no remote-tracking ref" if local.stdout.strip() else "" + return "a commit here is on no remote-tracking ref" if local.strip() else "" def idle_runner_checkout_veto(runner: Path, checkout: Path, opened: frozenset[str], cmds: str) -> str: @@ -5330,10 +5349,12 @@ LEGACY_POLICY = {"soft_idle_hours": 24.0, "stale_marker_max_hours": 48.0, "cmux_ def keep_legacy_ages() -> None: - """Put back the pre-12 h ages for this run (a reserved host). POLICY keys a disk-policy.json sets stay its own.""" + """Put back the pre-12 h ages for this run (a reserved host). A POLICY key disk-policy.json sets stays its own, + even one set to the new default; so does one already changed in this run.""" globals().update(LEGACY_AGES) + explicit = policy_overrides() for key, value in LEGACY_POLICY.items(): - if POLICY.get(key) == POLICY_DEFAULTS.get(key): + if key not in explicit and POLICY.get(key) == POLICY_DEFAULTS.get(key): POLICY[key] = value diff --git a/scripts/test-glaeda-disk.py b/scripts/test-glaeda-disk.py index ed113c97e..cb4b4bd6d 100644 --- a/scripts/test-glaeda-disk.py +++ b/scripts/test-glaeda-disk.py @@ -4511,6 +4511,18 @@ def git(*args: str) -> None: git("commit", "-q", "--allow-empty", "-m", "local only") (co / "build-output.o").write_text("x") # untracked build output never keeps a checkout + def _git_worktree_commit(self, co: Path) -> None: + """A detached linked worktree of co (inside its _work dir) with a commit no ref names.""" + env = {"PATH": "/usr/bin:/bin:/opt/homebrew/bin", "HOME": os.fspath(self.root), "GIT_AUTHOR_NAME": "t", + "GIT_AUTHOR_EMAIL": "t@x", "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@x"} + wt = co.parent / "linked" + if wt.exists(): + shutil.rmtree(wt) + subprocess.run(["git", "-C", os.fspath(co), "worktree", "add", "-q", "--detach", os.fspath(wt)], env=env, + check=True, capture_output=True) + subprocess.run(["git", "-C", os.fspath(wt), "commit", "-q", "--allow-empty", "-m", "only in the worktree"], + env=env, check=True, capture_output=True) + def test_stale_runner_checkouts_go_above_the_floor_and_without_one(self) -> None: """Old runner work dirs go by age: a checkout no job used for DEFAULT_AGE_HOURS goes whatever the free space, unless git would lose something (a commit on no remote-tracking ref) or holds a lock; younger ones stay.""" @@ -4518,7 +4530,8 @@ def test_stale_runner_checkouts_go_above_the_floor_and_without_one(self) -> None self.assertEqual(stale, 12.0) names = {"actions-runner-glaeda": stale + 1, "actions-runner-glaeda-2": stale + 30, "actions-runner-glaeda-3": stale - 1, "actions-runner-glaeda-4": stale + 2, - "actions-runner-glaeda-5": stale + 3} + "actions-runner-glaeda-5": stale + 3, "actions-runner-glaeda-6": stale + 3, + "actions-runner-glaeda-7": stale + 3, "actions-runner-glaeda-8": stale + 3} home, cos = self._idle_checkouts(names) gd.process_evidence = lambda: ([], "") for floor in (30.0, 0.0): @@ -4527,6 +4540,12 @@ def test_stale_runner_checkouts_go_above_the_floor_and_without_one(self) -> None self._git_checkout(cos[name], pushed=name != "actions-runner-glaeda-4") if name == "actions-runner-glaeda-5": (cos[name] / ".git/index.lock").write_text("") + if name == "actions-runner-glaeda-6": + (cos[name] / ".git/FETCH_HEAD.lock").write_text("") + if name == "actions-runner-glaeda-7": + (cos[name] / ".git/rebase-merge").mkdir() + if name == "actions-runner-glaeda-8": # a linked worktree holds the only copy of a commit + self._git_worktree_commit(cos[name]) self._age(home / name, hours=age) patches, removed = self._floor_disk(400.0) # far above the floor and its target receipt = self.receipt() @@ -4534,13 +4553,18 @@ def test_stale_runner_checkouts_go_above_the_floor_and_without_one(self) -> None self.assertGreater(gd.evict_idle_runner_checkouts(receipt, floor, home), 0) self.assertEqual(sorted(removed), sorted([str(cos["actions-runner-glaeda-2"]), str(cos["actions-runner-glaeda"])])) - for name in ("actions-runner-glaeda-3", "actions-runner-glaeda-4", "actions-runner-glaeda-5"): + for name in ("actions-runner-glaeda-3", "actions-runner-glaeda-4", "actions-runner-glaeda-5", + "actions-runner-glaeda-6", "actions-runner-glaeda-7", "actions-runner-glaeda-8"): self.assertTrue((cos[name] / ".git").is_dir(), name) self.assertIn("unused for 12 h", out.getvalue()) why = {Path(r["path"]).parts[-4]: (r["outcome"], r["reasons"][0]) for r in map(json.loads, receipt.read_text().splitlines())} self.assertEqual(why["actions-runner-glaeda-4"], ("blocked:git", "a commit here is on no remote-tracking ref")) self.assertEqual(why["actions-runner-glaeda-5"], ("blocked:git", "git holds a lock in it")) + self.assertEqual(why["actions-runner-glaeda-6"], ("blocked:git", "git holds a lock in it")) + self.assertEqual(why["actions-runner-glaeda-7"], ("blocked:git", "git holds a lock in it")) + self.assertEqual(why["actions-runner-glaeda-8"], + ("blocked:git", "a commit here is on no remote-tracking ref")) self.assertEqual(why["actions-runner-glaeda"], ("reclaimed", "unused for 12 h")) def test_every_disposable_age_is_twelve_hours(self) -> None: @@ -6320,6 +6344,18 @@ def test_a_reserved_host_keeps_the_ages_before_twelve_hours(self) -> None: self.assertEqual((gd.DEFAULT_AGE_HOURS, gd.CMUX_NEXT_CI_MIN_IDLE_HOURS, gd.RUNTIME_IDLE_DAYS), (24.0, 72.0, 14.0)) self.assertEqual(gd.RUNNER_STALE_CHECKOUT_HOURS, float("inf"), "no above-floor checkout eviction") + def test_a_disk_policy_key_set_to_the_new_default_stays_on_a_reserved_host(self) -> None: + saved = {n: getattr(gd, n) for n in gd.LEGACY_AGES} + self.addCleanup(lambda: [setattr(gd, n, v) for n, v in saved.items()]) + policy = Path(tempfile.mkdtemp()) / "disk-policy.json" + self.addCleanup(shutil.rmtree, policy.parent) + policy.write_text(json.dumps({"soft_idle_hours": 12, "stale_marker_max_hours": "twelve"})) + with mock.patch.object(gd, "DISK_POLICY", policy), mock.patch.dict(gd.POLICY, gd.disk_policy(policy)): + self.assertEqual(gd.policy_overrides(), {"soft_idle_hours": 12.0}) + gd.keep_legacy_ages() + self.assertEqual(gd.POLICY["soft_idle_hours"], 12.0, "set explicitly, even to the new default") + self.assertEqual(gd.POLICY["stale_marker_max_hours"], 48.0, "a wrongly typed key is not a setting") + def test_the_shipped_list_parses_and_matches_the_registry_where_it_is_checked_out(self) -> None: shipped = json.loads((ROOT / "config/fleet-reservations.json").read_text()) for e in shipped["reservations"]: From dd6b9974acfb33bbf458f3625f0272148ad4096c Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 26 Oct 2026 15:37:00 -0700 Subject: [PATCH 13/28] glaeda-disk: a fleet reservation list past its expires, or without one, reads as reserved The writer (leo-workspace host-baseline ci-minis) stamps expires 13 h after each write and refreshes it every 6 h, so a mini the writer stopped reaching (for one, a host reserved after its last write) falls back to reserved by itself. Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- scripts/glaeda-disk | 10 +++++++--- scripts/test-glaeda-disk.py | 21 ++++++++++++++++----- scripts/test-glaeda-fleet-cas-prune.py | 2 +- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/scripts/glaeda-disk b/scripts/glaeda-disk index ee873a8ac..06bec31fa 100755 --- a/scripts/glaeda-disk +++ b/scripts/glaeda-disk @@ -5217,9 +5217,11 @@ def runner_floor_gib(home: Path = HOME) -> float: # target_margin_gib, so state its holder keeps idle is not aged out. The list is never part of this repository: the # fleet's own writer puts it on each host, in the home of the user that owns FLEET_ROOT # (~/.local/share/glaeda/fleet-reservations.json, schema glaeda-fleet-reservations/v1: -# {"reservations": [{"id", "owner", "hosts": [...], "from", "until"}], "aliases": {host: [...]}}). The host-local -# reservation (FLEET_ROOT/reservation.json, glaeda-reservation/v1, the one the runner hook reads) counts too. Fails -# closed: a missing, unreadable or malformed list reads as reserved, whatever the host-local file says. +# {"expires", "reservations": [{"id", "owner", "hosts": [...], "from", "until"}], "aliases": {host: [...]}}; the +# writer stamps expires 13 h after each write and refreshes it every 6 h). The host-local reservation +# (FLEET_ROOT/reservation.json, glaeda-reservation/v1, the one the runner hook reads) counts too. Fails closed: a +# missing, unreadable, malformed or expired list, or one with no expires, reads as reserved, whatever the +# host-local file says. FLEET_RESERVATIONS: Path | None = None # None: fleet_reservations_path(); tests point it at a fixture FLEET_RESERVATIONS_NAME = ".local/share/glaeda/fleet-reservations.json" UPDATE_CONFIG = HOME / ".config/glaeda/update.json" @@ -5277,6 +5279,8 @@ def host_reservation(now: float | None = None, path: Path | None = None, names: entries, aliases = doc["reservations"], doc.get("aliases", {}) if not isinstance(entries, list) or not isinstance(aliases, dict): raise ValueError("malformed") + if reservation_time(doc["expires"]) <= now: # KeyError without one: the writer stopped refreshing it + return f"fleet reservations expired at {doc['expires']}; kept as reserved" alias = {str(h).lower(): {a.lower() for a in _str_list(v, "aliases")} for h, v in aliases.items()} hit = "" for e in entries: # every entry is checked, so one malformed entry anywhere fails closed diff --git a/scripts/test-glaeda-disk.py b/scripts/test-glaeda-disk.py index 248819cdd..b70a36a31 100644 --- a/scripts/test-glaeda-disk.py +++ b/scripts/test-glaeda-disk.py @@ -43,7 +43,8 @@ # Every test runs on an unreserved host unless it says otherwise (HostReservationTest): an empty reservation list, so a # test's host never reads the real ~/.local/share/glaeda/fleet-reservations.json, or fails closed without one. _NO_RESERVATIONS = tempfile.NamedTemporaryFile("w", suffix="-fleet-reservations.json", delete=False) -_NO_RESERVATIONS.write('{"schema": "glaeda-fleet-reservations/v1", "reservations": [], "aliases": {}}') +_NO_RESERVATIONS.write('{"schema": "glaeda-fleet-reservations/v1", "expires": "2999-01-01T00:00:00Z", ' + '"reservations": [], "aliases": {}}') _NO_RESERVATIONS.close() gd.FLEET_RESERVATIONS = Path(_NO_RESERVATIONS.name) gd.HOST_RESERVATION_LOCAL = Path(_NO_RESERVATIONS.name + ".none") @@ -6207,7 +6208,8 @@ def setUp(self) -> None: self.root = Path(self.tmp.name) self.list = self.root / "fleet-reservations.json" self.local = self.root / "reservation.json" - self.list.write_text(json.dumps({"schema": "glaeda-fleet-reservations/v1", "aliases": {"cmux30": ["cmux-thirty"]}, + self.list.write_text(json.dumps({"schema": "glaeda-fleet-reservations/v1", "expires": "2026-10-27T13:00:00Z", + "aliases": {"cmux30": ["cmux-thirty"]}, "reservations": [{"id": "r1", "owner": "someone", "hosts": ["cmux30"], "from": "2026-10-25T22:00:00Z", "until": "2026-10-28T23:59:59Z"}]})) @@ -6220,6 +6222,8 @@ def test_an_active_entry_or_alias_reserves_and_it_lapses_by_itself(self) -> None self.assertIn("reserved for someone until 2026-10-28T23:59:59Z (r1)", self.reserved({"cmux30"})) self.assertIn("reserved for someone", self.reserved({"CMUX-Thirty"})) self.assertEqual(self.reserved({"cmux31"}), "") + doc = json.loads(self.list.read_text()) + self.list.write_text(json.dumps(dict(doc, expires="2026-10-30T00:00:00Z"))) self.assertEqual(self.reserved({"cmux30"}, gd.reservation_time("2026-10-29T00:00:00Z")), "", "lapsed") self.assertEqual(self.reserved({"cmux30"}, gd.reservation_time("2026-10-25T21:59:59Z")), "", "not yet") @@ -6253,11 +6257,18 @@ def test_a_malformed_list_anywhere_fails_closed(self) -> None: {"reservations": [good], "aliases": []}, ) for doc in bad_docs: - with self.subTest(doc=doc): - self.list.write_text(json.dumps(doc)) + with self.subTest(doc=doc): # each with a fresh expires, so only its own fault can make it fail + self.list.write_text(json.dumps(dict(doc, expires="2026-10-27T13:00:00Z") if isinstance(doc, dict) + else doc)) self.assertIn("unreadable", self.reserved({"cmux31"})) - self.list.write_text(json.dumps({"reservations": [good]})) + fresh = "2026-10-27T13:00:00Z" + self.list.write_text(json.dumps({"reservations": [good], "expires": fresh})) self.assertEqual(self.reserved({"cmux31"}), "", "a well-formed list without this host") + for expires in (None, "2026-10-26T23:59:59Z", "soon", 12): + with self.subTest(expires=expires): + doc = {"reservations": [good]} if expires is None else {"reservations": [good], "expires": expires} + self.list.write_text(json.dumps(doc)) + self.assertIn("kept as reserved", self.reserved({"cmux31"})) def test_a_missing_list_is_reserved_whatever_the_host_local_file_says(self) -> None: now = int(self.during) diff --git a/scripts/test-glaeda-fleet-cas-prune.py b/scripts/test-glaeda-fleet-cas-prune.py index 31b82b9b5..776c34e72 100644 --- a/scripts/test-glaeda-fleet-cas-prune.py +++ b/scripts/test-glaeda-fleet-cas-prune.py @@ -208,7 +208,7 @@ def test_reserved_above_floor_reads_glaeda_disk_and_fails_closed(self) -> None: # the real glaeda-disk loads (its dataclasses need the module in sys.modules) and answers from a made-up list gd = fp.load_glaeda_disk(ROOT / "scripts/glaeda-disk") listing = Path(self.tmp.name) / "fleet-reservations.json" - listing.write_text(json.dumps({"reservations": [{"id": "r9", "owner": "someone", "hosts": ["cmux30"], + listing.write_text(json.dumps({"expires": "2999-01-01T00:00:00Z", "reservations": [{"id": "r9", "owner": "someone", "hosts": ["cmux30"], "from": "2000-01-01T00:00:00Z", "until": "2999-01-01T00:00:00Z"}]})) gd.FLEET_RESERVATIONS = listing From 59e38c7b59c6050b28fb00a6e9d23cda79099106 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 26 Oct 2026 15:52:30 -0700 Subject: [PATCH 14/28] glaeda-disk: a reserved host runs the pressure pass and nothing else; markers and intervals fail closed Review findings on #1626: - a FLEET_ROOT/reservation.json that is there but unreadable, not glaeda-reservation/v1 or without integer since/until holds the host, as host_busy reads it (absent: no hold); - a fleet entry with from >= until fails the whole list closed; - on a reserved host, after the floor-pressure pass, no stale-marker clearing, paused-work cleanup, test-residue reaping, home-clone gc or runner repack runs, and --dedupe / --dedupe-builds do nothing (the reservation is asked for them too); - filesystems() gets reserved= only when set, so callers and mocks that pass positionals keep working. Each gate has a test that fails when its check is removed. Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- scripts/glaeda-disk | 36 +++++++++++++++++++-------- scripts/test-glaeda-disk.py | 49 +++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 10 deletions(-) diff --git a/scripts/glaeda-disk b/scripts/glaeda-disk index 06bec31fa..5abd2c312 100755 --- a/scripts/glaeda-disk +++ b/scripts/glaeda-disk @@ -5289,16 +5289,27 @@ def host_reservation(now: float | None = None, path: Path | None = None, names: hosts = {h.lower() for h in _str_list(e["hosts"], "hosts")} hosts |= {a for h in list(hosts) for a in alias.get(h, ())} start, end = reservation_time(e["from"]), reservation_time(e["until"]) + if start >= end: + raise ValueError("from must be before until") if not hit and names & hosts and start <= now <= end: hit = f"reserved for {e.get('owner', '?')} until {e['until']} ({e.get('id', '?')})" if hit: return hit except (OSError, ValueError, KeyError, TypeError) as error: return f"fleet reservations unreadable ({type(error).__name__}); kept as reserved" - with contextlib.suppress(OSError, ValueError, TypeError, AttributeError, KeyError): - held = json.loads((HOST_RESERVATION_LOCAL if local is None else local).read_text()) - if held.get("schema") == "glaeda-reservation/v1" and int(held["since"]) <= now < int(held["until"]): - return f"host reservation by {held.get('owner', '?')} until {held['until']}" + marker = HOST_RESERVATION_LOCAL if local is None else local + if not os.path.lexists(marker): + return "" + try: # a marker that is there but cannot be read or understood holds the host, as host_busy reads it + held = json.loads(marker.read_text()) + since, until = held["since"], held["until"] + if (held.get("schema") != "glaeda-reservation/v1" or not all( + isinstance(v, int) and not isinstance(v, bool) for v in (since, until))): + raise ValueError("not a glaeda-reservation/v1 marker") + except (OSError, ValueError, TypeError, AttributeError, KeyError) as error: + return f"host reservation marker unreadable ({type(error).__name__}); kept as reserved" + if since <= now < until: + return f"host reservation by {held.get('owner', '?')} until {until}" return "" @@ -7798,7 +7809,8 @@ def _main_impl(argv: list[str] | None = None) -> int: a.target = f"{a.need + POLICY['target_margin_gib']:g}" # before any family is built: a reserved host's families keep its holder's ages (keep_legacy_ages, once 12 h lands) - reserved = host_reservation() if role == "fleet-mini" and DARWIN and a.apply else "" + reserved = (host_reservation() if role == "fleet-mini" and DARWIN and (a.apply or a.dedupe or a.dedupe_builds) + else "") global RUNNER_CACHE_PROFILE RUNNER_CACHE_PROFILE = bool(a.runner_caches) fams = runner_cache_families() if RUNNER_CACHE_PROFILE else default_families(role) @@ -7818,6 +7830,9 @@ def _main_impl(argv: list[str] | None = None) -> int: if RUNNER_CACHE_PROFILE and not runner_profile_roots_safe(fams): print("runner cache profile roots are not safe; skipped") return 0 + if (a.dedupe or a.dedupe_builds) and reserved: + print(f"{reserved}: only floor-pressure eviction runs here; no dedupe") + return 0 if a.dedupe or a.dedupe_builds: dd = next((f.root for f in fams if f.id == "xcode-derived-data"), None) ci_root = HOME / "Library/Caches" / CMUX_NEXT_CI_ROOT @@ -7970,7 +7985,8 @@ def _main_impl(argv: list[str] | None = None) -> int: fcntl.flock(maintenance_handle, fcntl.LOCK_UN) maintenance_handle.close() return result - stale_marker_released = clear_stale_build_markers(a.receipt) + if not reserved: # a reserved host gets the floor-pressure pass and nothing else + stale_marker_released = clear_stale_build_markers(a.receipt) idle = a.idle_hours if a.idle_hours is not None else IDLE_SWEEP_HOURS if sweep else (6 if under else 24) at, known, measured_at = (0.0, {}, {}) if a.refresh or a.no_snapshot else load_snapshot_meta() original_known = set(known) @@ -8098,10 +8114,10 @@ def _main_impl(argv: list[str] | None = None) -> int: logical_released = stale_marker_released + apply_pass([i for i in items if id(i) not in last]) # A drained runner is explicitly held by cmux_mini_fix; once its listener and worker # are absent, its disposable checkout can be cleared under this same eviction lock. - if not RUNNER_CACHE_PROFILE: + if not RUNNER_CACHE_PROFILE and not reserved: logical_released += cleanup_paused_runner_work(a.receipt) # test scratch left in $TMPDIR, two days untouched: too small per entry for any family's size floor - if not RUNNER_CACHE_PROFILE and not a.cmux_next_drain \ + if not RUNNER_CACHE_PROFILE and not a.cmux_next_drain and not reserved \ and (not a.family or TMP_TEST_RESIDUE_FAMILY in a.family): logical_released += reap_tmpdir_test_residue(a.receipt, halt=halt) # still under the runner gate's floor after every family: idle runners' whole checkouts, last @@ -8113,9 +8129,9 @@ def _main_impl(argv: list[str] | None = None) -> int: # a home clone that fetch after fetch left in many packs: repacked at most hourly or under pressure, # after the eviction lock is released, so a gc of many minutes never holds back another eviction home_clone_gc_due = ((sweep or a.pressure) and ROLE == "fleet-mini" and not RUNNER_CACHE_PROFILE - and not a.cmux_next_drain and (not a.family or HOME_CLONE_GC_FAMILY in a.family)) + and not a.cmux_next_drain and not reserved and (not a.family or HOME_CLONE_GC_FAMILY in a.family)) runner_repack_due = ((sweep or a.pressure) and ROLE == "fleet-mini" and not RUNNER_CACHE_PROFILE - and not a.cmux_next_drain + and not a.cmux_next_drain and not reserved and (not a.family or RUNNER_CHECKOUT_REPACK_FAMILY in a.family)) remaining = [] for fs in shown: diff --git a/scripts/test-glaeda-disk.py b/scripts/test-glaeda-disk.py index b70a36a31..479e20c0c 100644 --- a/scripts/test-glaeda-disk.py +++ b/scripts/test-glaeda-disk.py @@ -6253,6 +6253,8 @@ def test_a_malformed_list_anywhere_fails_closed(self) -> None: {"reservations": [dict(good, hosts=[31])]}, {"reservations": ["cmux31"]}, {"reservations": [dict(good, until="soon")]}, # a bad time on an entry naming another host + {"reservations": [dict(good, hosts=["cmux31"], until=good["from"])]}, # zero length + {"reservations": [dict(good, hosts=["cmux31"], **{"from": good["until"], "until": good["from"]})]}, {"reservations": [good], "aliases": {"cmux40": "cmux-forty"}}, {"reservations": [good], "aliases": []}, ) @@ -6300,6 +6302,53 @@ def test_a_reserved_hosts_pressure_starts_at_the_floor_itself(self) -> None: self.assertEqual(held.low, 30 * gd.GIB) self.assertEqual(held.target, int((30 + gd.POLICY["target_margin_gib"]) * gd.GIB)) + def test_a_local_marker_that_is_there_but_unreadable_holds_the_host(self) -> None: + now = int(self.during) + for text in ("{nope", "[]", json.dumps({"schema": "other", "since": now - 60, "until": now + 60}), + json.dumps({"schema": "glaeda-reservation/v1", "since": "yesterday", "until": now + 60}), + json.dumps({"schema": "glaeda-reservation/v1", "until": now + 60})): + with self.subTest(text=text): + self.local.write_text(text) + self.assertIn("marker unreadable", self.reserved({"cmux31"})) + self.local.unlink() + self.assertEqual(self.reserved({"cmux31"}), "", "no marker, no hold") + + def test_a_reserved_pressure_pass_does_nothing_but_evict(self) -> None: + """Under its floor, a reserved host runs the pressure pass and nothing else: no marker clearing, paused-work + cleanup, test-residue reaping, clone gc or repack; and no dedupe at all.""" + empty = self.root / "empty" + empty.mkdir() + state = self.root / "state" + state.mkdir() + why = "reserved for someone until 2026-10-28T23:59:59Z (r1)" + with contextlib.ExitStack() as stack: + for name in ("SNAPSHOT", "DEDUPE_STATE", "CMUX_NEXT_CAP_RECORD", "MEASURE_LOCK", "ACCOUNTING", + "HEALTH_SUMMARY", "EVICT_LOCK", "HOOK_PASS_LOCK", "HOOK_PASS_PENDING", "EVICT_OWNER", + "NESTED_KEEP"): + stack.enter_context(mock.patch.object(gd, name, state / name)) + for name in ("clear_stale_build_markers", "cleanup_paused_runner_work", "reap_tmpdir_test_residue", + "gc_home_clones", "repack_runner_checkouts", "dedupe_xcode", "dedupe_captures"): + stack.enter_context(mock.patch.object(gd, name, side_effect=AssertionError(f"{name} ran"))) + stack.enter_context(mock.patch.object(gd, "default_families", + return_value=[gd.Family("tmp", empty, True, "scratch")])) + stack.enter_context(mock.patch.object(gd, "host_reservation", return_value=why)) + stack.enter_context(mock.patch.object(gd, "runner_floor_gib", return_value=30.0)) + stack.enter_context(mock.patch.object(gd, "free_bytes", return_value=(20 * gd.GIB, 1000 * gd.GIB))) + stack.enter_context(mock.patch.object(gd, "DARWIN", True)) + stack.enter_context(mock.patch.object(gd, "disk_off", return_value="")) + stack.enter_context(mock.patch.object(gd, "notify_disk")) + evict = stack.enter_context(mock.patch.object(gd, "evict_idle_runner_checkouts", return_value=0)) + stack.enter_context(mock.patch.dict(gd.RUN, clear=True)) + out = stack.enter_context(contextlib.redirect_stdout(io.StringIO())) + for args in (("--pressure",), ("--dedupe",), ("--dedupe-builds",)): + with self.subTest(args=args): + code = gd._main_impl(["--apply", "--top", "0", "--no-snapshot", + "--receipt", os.fspath(self.root / "receipt.jsonl"), *args]) + self.assertEqual(code, 0, out.getvalue()) + self.assertIn("eviction summary", out.getvalue(), "the pressure pass itself ran") + evict.assert_called_once() # the floor's last resort still runs + self.assertIn("no dedupe", out.getvalue()) + def run_disk(self, *args: str, reserved: str, floor: float, free_gib: float): """glaeda-disk's main on a fake fleet mini; returns (exit code, output, the families' low/target it used).""" seen, run = {}, {} From 6a5a998f49d63f13ac0b0dd4e96410c9035a990a Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 26 Oct 2026 15:53:04 -0700 Subject: [PATCH 15/28] glaeda-disk: the runner checkout git veto drops the caller's GIT_* and keeps what it cannot read Review findings on #1622: GIT_DIR/GIT_WORK_TREE from the caller would point git -C at another repository (as _checkout_git already strips them); an unreadable directory under .git (a hidden worktree's lock) now raises in the walk and keeps the checkout. Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- scripts/glaeda-disk | 9 +++++++-- scripts/test-glaeda-disk.py | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/scripts/glaeda-disk b/scripts/glaeda-disk index 2450702c9..65236e088 100755 --- a/scripts/glaeda-disk +++ b/scripts/glaeda-disk @@ -1963,8 +1963,11 @@ def runner_checkout_git_veto(checkout: Path) -> str: dotgit = checkout / ".git" if dotgit.is_symlink() or not dotgit.is_dir(): return "not a plain checkout (.git is not a directory)" + def unreadable(error: OSError) -> None: + raise error # a directory git keeps state in that cannot be read is doubt, and doubt keeps it + try: - for root, dirs, files in os.walk(dotgit): + for root, dirs, files in os.walk(dotgit, onerror=unreadable): rel = Path(root).relative_to(dotgit).parts dirs[:] = [d for d in dirs if d not in ("objects", "lfs", "logs")] if any(n in GIT_LOCK_NAMES for n in (*files, *dirs)) or any( @@ -1972,7 +1975,9 @@ def runner_checkout_git_veto(checkout: Path) -> str: return "git holds a lock in it" except OSError: return "git could not say whether it holds a lock" - env = {**os.environ, "GIT_OPTIONAL_LOCKS": "0"} + # the caller's GIT_* would override -C (a hook's GIT_DIR names one repository for every checkout) + env = {k: v for k, v in os.environ.items() if not k.startswith("GIT_")} + env.update(GIT_OPTIONAL_LOCKS="0", GIT_TERMINAL_PROMPT="0") def git(*args: str, timeout: int = 60) -> str: return subprocess.run(["git", "-C", os.fspath(checkout), *args], capture_output=True, text=True, diff --git a/scripts/test-glaeda-disk.py b/scripts/test-glaeda-disk.py index 06354d3f8..fece38178 100644 --- a/scripts/test-glaeda-disk.py +++ b/scripts/test-glaeda-disk.py @@ -4569,6 +4569,25 @@ def test_stale_runner_checkouts_go_above_the_floor_and_without_one(self) -> None ("blocked:git", "a commit here is on no remote-tracking ref")) self.assertEqual(why["actions-runner-glaeda"], ("reclaimed", "unused for 12 h")) + def test_the_git_veto_ignores_the_callers_git_env_and_keeps_what_it_cannot_read(self) -> None: + co = self.root / "veto/_work/cmux/cmux" + self._git_checkout(co, pushed=False) + other = self.root / "other" + self._git_checkout(other, pushed=True) + with mock.patch.dict(os.environ, {"GIT_DIR": os.fspath(other / ".git"), "GIT_WORK_TREE": os.fspath(other)}): + self.assertEqual(gd.runner_checkout_git_veto(co), "a commit here is on no remote-tracking ref") + self._git_checkout(co, pushed=True) + self.assertEqual(gd.runner_checkout_git_veto(co), "") + locked = co / ".git/worktrees/hidden" + locked.mkdir(parents=True) + locked.chmod(0) + try: # tearDown removes the tree before any addCleanup runs, so the mode goes back here + if os.access(locked, os.R_OK): + self.skipTest("running as a user who can read a mode-0 directory") + self.assertEqual(gd.runner_checkout_git_veto(co), "git could not say whether it holds a lock") + finally: + locked.chmod(0o755) + def test_every_disposable_age_is_twelve_hours(self) -> None: """Leo, 2026-10-26: "When have we ever had something useful greater than 12 hours?" Only the newest warm compile cache per builder (the per-host cmux-next root) is exempt, by role.""" From 9d8dd5c6bf3cecc16208e6429dae1910da738f19 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 26 Oct 2026 15:56:02 -0700 Subject: [PATCH 16/28] glaeda-disk: mirror bundling finds nested submodules, rediscovers repos and rechecks owners before delete Addresses the second Codex review of #1625: 1. mirror_repos queues every initialized submodule work tree (embedded or in modules/), so a submodule's own submodules are bundled too. 2. and 3. mirror_moved finds the item's repositories again right before the delete: a repository created meanwhile, or a work tree no longer clean (hidden-file flags included), keeps the tree. 4. mirror_moved re-reads owner_retired and the fleet artifact references after bundling. 5. A locked linked worktree keeps the tree, as in git_loss. Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- scripts/glaeda-disk | 98 ++++++++++++++++++++++++------------- scripts/test-glaeda-disk.py | 83 ++++++++++++++++++++++++++++++- 2 files changed, 146 insertions(+), 35 deletions(-) diff --git a/scripts/glaeda-disk b/scripts/glaeda-disk index c0c22cfb0..f7fb17256 100755 --- a/scripts/glaeda-disk +++ b/scripts/glaeda-disk @@ -4633,52 +4633,59 @@ def mirror_git_snapshot(gitdirs: list[Path], trees: list[Path]) -> list[str | No return snap -def archive_mirror(path: Path) -> tuple[str, list[Path], tuple[list[Path], list[Path], list[str | None]]]: - """Before a mirror item goes: bundle_unpushed for every repository it takes along, found as _nested_loss - finds them (checkouts, bare repositories, a linked worktree's repository inside it, checkouts in ignored - directories, submodule stores and embedded submodules). A tree with uncommitted work, or whose repositories - cannot all be found or bundled, is kept. ("" or why it must stay, the bundles written, what to recheck).""" - none: tuple[list[Path], list[Path], list[str | None]] = ([], [], []) +def mirror_repos(path: Path) -> tuple[str, dict[str, Path], list[Path]]: + """Every repository a mirror item takes along, found as _nested_loss finds them (checkouts, bare + repositories, a linked worktree's repository inside it, checkouts in ignored directories, submodule stores + and initialized submodules, recursively), and its work trees, each checked clean right now. + ("" or why it must stay, real git dir -> git dir, work trees).""" queue = nested_checkouts(path) if queue is None: - return "too deep or large to find every checkout", [], none + return "too deep or large to find every checkout", {}, [] queue, inside = list(queue), os.path.realpath(path) + "/" gitdirs: dict[str, Path] = {} trees: list[Path] = [] + seen: set[str] = set() while queue: root = queue.pop() + if (real := os.path.realpath(root)) in seen: + continue + seen.add(real) where, dotgit = root.relative_to(path), root / ".git" if not os.path.lexists(dotgit): - gitdirs[os.path.realpath(root)] = root + gitdirs[real] = root continue if dotgit.is_symlink(): - return f"{where}: .git is a symlink", [], none + return f"{where}: .git is a symlink", {}, [] if (dirty := _work_tree_dirty(root)): - return f"uncommitted: {where}: {dirty}", [], none + return f"uncommitted: {where}: {dirty}", {}, [] trees.append(root) if (inner := _ignored_checkouts(root)) is None: - return f"{where}: too deep or large to find every checkout in its ignored files", [], none + return f"{where}: too deep or large to find every checkout in its ignored files", {}, [] queue += inner - if dotgit.is_file(): # a linked worktree: bundle its repository when that goes too - common, own = git_common_dir(root), _git_dir_run(dotgit, "rev-parse", "--absolute-git-dir") - if not common or own is None: - return f"{where}: unreadable worktree", [], none - if (os.path.realpath(common) + "/").startswith(inside): - gitdirs[os.path.realpath(common)] = Path(common) - continue - tips = local_tips(Path(own.strip()), worktree_only=True) - left = None if tips is None else _git_dir_run( - Path(common), "rev-list", "-n1", *tips, "--not", "--branches", "--tags", "--remotes") if tips else "" - if left is None or left.strip(): - return f"{where}: commits only this worktree's reflog holds", [], none - continue - gitdirs[os.path.realpath(dotgit)] = dotgit + own = _git_dir_run(dotgit, "rev-parse", "--absolute-git-dir") links = _git_dir_run(dotgit, "--work-tree", str(root), "ls-files", "-s") - if links is None: - return f"{where}: unreadable index", [], none - for ln in links.splitlines(): - if ln.startswith("160000 ") and (sub := root / ln.split("\t", 1)[1] / ".git").is_dir(): - gitdirs[os.path.realpath(sub)] = sub + if own is None or links is None: + return f"{where}: unreadable git dir or index", {}, [] + # initialized submodules, embedded (.git a directory) or stored in modules/ (.git a file), and theirs + queue += [root / ln.split("\t", 1)[1] for ln in links.splitlines() + if ln.startswith("160000 ") and os.path.lexists(root / ln.split("\t", 1)[1] / ".git")] + if dotgit.is_dir(): + gitdirs[os.path.realpath(dotgit)] = dotgit + continue + # a .git file: a linked worktree, or a submodule whose repository is in its superproject's modules/ + common = git_common_dir(root) + if not common: + return f"{where}: unreadable worktree", {}, [] + if (os.path.realpath(common) + "/").startswith(inside): + gitdirs[os.path.realpath(common)] = Path(common) + continue + if os.path.lexists(os.path.join(own.strip(), "locked")): + return f"{where}: locked worktree", {}, [] + tips = local_tips(Path(own.strip()), worktree_only=True) + left = None if tips is None else _git_dir_run( + Path(common), "rev-list", "-n1", *tips, "--not", "--branches", "--tags", "--remotes") if tips else "" + if left is None or left.strip(): + return f"{where}: commits only this worktree's reflog holds", {}, [] unreadable: list[str] = [] for g in list(gitdirs.values()): # submodule stores: modules/, nested ones too for dirpath, dirnames, _ in os.walk(g / "modules", onerror=lambda e: unreadable.append(str(e.filename))): @@ -4686,19 +4693,33 @@ def archive_mirror(path: Path) -> tuple[str, list[Path], tuple[list[Path], list[ gitdirs[os.path.realpath(dirpath)] = Path(dirpath) dirnames[:] = ["modules"] if "modules" in dirnames else [] if [u for u in unreadable if os.path.lexists(u)]: # a missing modules/ is no submodule store - return f"unreadable submodule store {unreadable[0]}", [], none + return f"unreadable submodule store {unreadable[0]}", {}, [] + for key, g in gitdirs.items(): + if any(t.exists() and not (os.path.realpath(t) + "/").startswith(inside) for t in linked_work_trees(g)): + return f"{key}: a worktree outside this item uses it", {}, [] + return "", gitdirs, sorted(trees) + + +def archive_mirror(path: Path) -> tuple[str, list[Path], tuple[list[Path], list[Path], list[str | None]]]: + """Before a mirror item goes: bundle_unpushed for every repository mirror_repos finds. A tree with + uncommitted work, or whose repositories cannot all be found or bundled, is kept. + ("" or why it must stay, the bundles written, what mirror_moved rechecks).""" + none: tuple[list[Path], list[Path], list[str | None]] = ([], [], []) + why, gitdirs, trees = mirror_repos(path) + if why: + return why, [], none snap = mirror_git_snapshot(list(gitdirs.values()), trees) bundles: list[Path] = [] for key, g in sorted(gitdirs.items()): - if any(t.exists() and not (os.path.realpath(t) + "/").startswith(inside) for t in linked_work_trees(g)): - return f"{key}: a worktree outside this item uses it", bundles, none if (why := bundle_unpushed(g, key, bundles)): return why, bundles, none return "", bundles, (list(gitdirs.values()), trees, snap) def mirror_moved(fam: Family, path: Path, recheck: tuple[list[Path], list[Path], list[str | None]]) -> str: - """Right before a bundled mirror goes: fresh process evidence, and its repositories as at bundle time.""" + """Right before a bundled mirror goes, everything the bundling took time over is read again: process + evidence, the family's owner and artifact references, and its repositories, found afresh (the same set, + every work tree clean) and as at bundle time.""" try: opened, cmds = process_evidence() except NoEvidence as e: @@ -4706,7 +4727,16 @@ def mirror_moved(fam: Family, path: Path, recheck: tuple[list[Path], list[Path], if in_cwd(os.fspath(path), open_index(opened)) or named_by(os.fspath(path), cmds) or holds_socket(path) \ or job_busy(fam, path): return "in use while its git work was bundled" + if fam.retired_only and not owner_retired(fam): + return "its owner loaded while its git work was bundled" + if fam.reference_root is not None and artifact_referenced(path, fresh_artifact_references(fam.reference_root)): + return "referenced while its git work was bundled" gitdirs, trees, snap = recheck + _NESTED_WALKS.pop(str(path), None) + why, now, now_trees = mirror_repos(path) + if why or {os.path.realpath(g) for g in now.values()} != {os.path.realpath(g) for g in gitdirs} \ + or now_trees != trees: + return f"its repositories changed while they were bundled{': ' + why if why else ''}" if None in snap or mirror_git_snapshot(gitdirs, trees) != snap: return "its git refs or work trees changed while they were bundled" return "" diff --git a/scripts/test-glaeda-disk.py b/scripts/test-glaeda-disk.py index 34ddc64de..a6529be96 100644 --- a/scripts/test-glaeda-disk.py +++ b/scripts/test-glaeda-disk.py @@ -1518,7 +1518,8 @@ def git_state(f, path): # a cargo target dir too large to search mock.patch.object(gd, "process_evidence", return_value=late), \ mock.patch.object(gd, "command_lines", return_value=late[1]), \ mock.patch.object(gd, "HOME", self.root / "home"), \ - mock.patch.object(gd, "archive_mirror", return_value=("", [], ([], [], []))): # bundling: the cmux-job tests + mock.patch.object(gd, "archive_mirror", return_value=("", [], ([], [], []))), \ + mock.patch.object(gd, "mirror_moved", return_value=""): # bundling: the cmux-job tests gd.apply(reclaim, {fam.id: fam}, receipt, None, 24) self.assertEqual(sorted(p.name for p in cache.iterdir()), ["cmux-tui-target", "fresh-workspaces", "leased", "swiftpm"]) @@ -5819,6 +5820,86 @@ def racing(path): self.assertIn("its git refs or work trees changed while they were bundled", rec["reasons"]) self.assertTrue(clone.is_dir()) + def bundled_pairs(self) -> set[tuple[str, str]]: + pairs = set() + for b in (self.home / "archive/glaeda-disk").glob("*.bundle"): + out = subprocess.run(["git", "bundle", "list-heads", str(b)], capture_output=True, text=True, check=True) + pairs |= {(ln.split(" ")[1], ln.split(" ")[0]) for ln in out.stdout.splitlines()} + return pairs + + def test_a_nested_embedded_submodules_commits_are_bundled(self) -> None: + unit, clone, env = self.mirror_unit() + sub, nested = clone / "sub", clone / "sub/nested" + for repo in (sub, nested): + subprocess.run(["git", "init", "-q", "-b", "main", str(repo)], check=True, env=env) + self.git(nested, "commit", "-q", "--allow-empty", "-m", "nested only", env=env) + head = self.git(nested, "rev-parse", "HEAD") + for outer, inner in ((sub, "nested"), (clone, "sub")): + self.git(outer, "add", inner, env=env) + self.git(outer, "commit", "-q", "-m", f"link {inner}", env=env) + rec, _ = self.apply_mirror(unit) + self.assertEqual(rec["outcome"], "reclaimed", rec["reasons"]) + self.assertIn(("refs/heads/main", head), self.bundled_pairs()) + + def test_a_repository_created_while_bundling_keeps_the_mirror(self) -> None: + unit, _, env = self.mirror_unit() + real = gd.archive_mirror + + def racing(path): + out = real(path) + subprocess.run(["git", "init", "-q", str(unit / "slot-2")], check=True, env=env) + self.git(unit / "slot-2", "commit", "-q", "--allow-empty", "-m", "new", env=env) + return out + with mock.patch.object(gd, "archive_mirror", racing): + rec, _ = self.apply_mirror(unit) + self.assertEqual(rec["outcome"], "changed:in-use") + self.assertTrue(any("repositories changed" in r for r in rec["reasons"])) + self.assertTrue((unit / "slot-2/.git").is_dir()) + + def test_a_tree_dirtied_before_its_snapshot_is_kept(self) -> None: + unit, clone, _ = self.mirror_unit() + real = gd.mirror_git_snapshot + calls = [] + + def dirtying(gitdirs, trees): + if not calls: # after the clean check, before the baseline: the baseline itself is dirty + (clone / "notes.txt").write_text("edited") + calls.append(1) + return real(gitdirs, trees) + with mock.patch.object(gd, "mirror_git_snapshot", dirtying): + rec, _ = self.apply_mirror(unit) + self.assertEqual(rec["outcome"], "changed:in-use") + self.assertTrue((clone / "notes.txt").exists()) + + def test_a_mirrors_owner_and_references_are_read_again_after_bundling(self) -> None: + unit, _, _ = self.mirror_unit() + why, _, recheck = gd.archive_mirror(unit) + self.assertEqual(why, "") + retired = gd.Family("hq", unit.parent, True, "x", retired_only=True) + referenced = gd.Family("hq", unit.parent, True, "x", reference_root=self.home) + with mock.patch.object(gd, "process_evidence", return_value=([], "")): + with mock.patch.object(gd, "owner_retired", return_value=False): + self.assertIn("owner loaded", gd.mirror_moved(retired, unit, recheck)) + with mock.patch.object(gd, "artifact_referenced", return_value=True): + self.assertIn("referenced", gd.mirror_moved(referenced, unit, recheck)) + with mock.patch.object(gd, "owner_retired", return_value=True), \ + mock.patch.object(gd, "artifact_referenced", return_value=False): + self.assertEqual(gd.mirror_moved(retired, unit, recheck), "") + + def test_a_locked_linked_worktree_keeps_the_mirror(self) -> None: + env = {**os.environ, "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", + "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"} + main = self.home / "main-clone" + subprocess.run(["git", "init", "-q", "-b", "main", str(main)], check=True, env=env) + self.git(main, "commit", "-q", "--allow-empty", "-m", "c", env=env) + unit = self.home / ".cache/cmux-job/reload-cloud/cmux-base" + unit.mkdir(parents=True) + self.git(main, "worktree", "add", "-q", "--detach", "--lock", str(unit / "slot-1"), env=env) + rec, _ = self.apply_mirror(unit) + self.assertEqual(rec["outcome"], "blocked:bundle-failed") + self.assertTrue(any("locked worktree" in r for r in rec["reasons"])) + self.assertTrue((unit / "slot-1/.git").is_file()) + def test_only_manifest_listed_bundles_at_their_hash_are_pruned_after_the_evidence_gate(self) -> None: archive = self.home / "archive/glaeda-disk" archive.mkdir(parents=True) From f49c6f3426cb14ff39e99d7f3b501f9c1e45a4ac Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 26 Oct 2026 15:58:49 -0700 Subject: [PATCH 17/28] glaeda-disk: pass reserved= to filesystems() only when set (the previous commit lost this edit) Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- scripts/glaeda-disk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/glaeda-disk b/scripts/glaeda-disk index 5abd2c312..7f60b013e 100755 --- a/scripts/glaeda-disk +++ b/scripts/glaeda-disk @@ -7870,7 +7870,7 @@ def _main_impl(argv: list[str] | None = None) -> int: a.low, a.target = f"{floor_gib:g}", f"{floor_gib + POLICY['target_margin_gib']:g}" print(f"{reserved}: only floor-pressure eviction runs here") fss = filesystems(fams or [Family("home", HOME, False, "")], a.low, a.target, - a.tmpfs_low, a.tmpfs_target, floor_gib, reserved=bool(reserved)) + a.tmpfs_low, a.tmpfs_target, floor_gib, **({"reserved": True} if reserved else {})) space(a.emergency, GIB) except ValueError: ap.error(f"--low/--target/--tmpfs-*/--emergency take GiB or N%: {a.low!r}, {a.target!r}, " From bae894e7ac24e5e86da801d6a9d044eec52b4c0c Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 26 Oct 2026 15:59:03 -0700 Subject: [PATCH 18/28] glaeda-disk: pass reserved= to filesystems() only when set Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- scripts/glaeda-disk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/glaeda-disk b/scripts/glaeda-disk index dfc8fbea1..51cd914e4 100755 --- a/scripts/glaeda-disk +++ b/scripts/glaeda-disk @@ -7965,7 +7965,7 @@ def _main_impl(argv: list[str] | None = None) -> int: a.low, a.target = f"{floor_gib:g}", f"{floor_gib + POLICY['target_margin_gib']:g}" print(f"{reserved}: only floor-pressure eviction runs here") fss = filesystems(fams or [Family("home", HOME, False, "")], a.low, a.target, - a.tmpfs_low, a.tmpfs_target, floor_gib, reserved=bool(reserved)) + a.tmpfs_low, a.tmpfs_target, floor_gib, **({"reserved": True} if reserved else {})) space(a.emergency, GIB) except ValueError: ap.error(f"--low/--target/--tmpfs-*/--emergency take GiB or N%: {a.low!r}, {a.target!r}, " From b84c765dc354d2726505e72bc40b4be0aac08103 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 26 Oct 2026 16:04:44 -0700 Subject: [PATCH 19/28] glaeda-disk: a local marker that cannot be read holds the host; a future since holds, as the runner hook reads it Review findings on #1626: lexists() read a permission error as absence; only FileNotFoundError on the read is absence now. The runner hook holds while now < until whatever since says; host_reservation matches it. Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- scripts/glaeda-disk | 10 ++++++---- scripts/test-glaeda-disk.py | 10 ++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/scripts/glaeda-disk b/scripts/glaeda-disk index 7f60b013e..1c870e341 100755 --- a/scripts/glaeda-disk +++ b/scripts/glaeda-disk @@ -5298,17 +5298,19 @@ def host_reservation(now: float | None = None, path: Path | None = None, names: except (OSError, ValueError, KeyError, TypeError) as error: return f"fleet reservations unreadable ({type(error).__name__}); kept as reserved" marker = HOST_RESERVATION_LOCAL if local is None else local - if not os.path.lexists(marker): - return "" try: # a marker that is there but cannot be read or understood holds the host, as host_busy reads it - held = json.loads(marker.read_text()) + try: + text = marker.read_text() + except FileNotFoundError: + return "" # only a marker that is certainly absent holds nothing + held = json.loads(text) since, until = held["since"], held["until"] if (held.get("schema") != "glaeda-reservation/v1" or not all( isinstance(v, int) and not isinstance(v, bool) for v in (since, until))): raise ValueError("not a glaeda-reservation/v1 marker") except (OSError, ValueError, TypeError, AttributeError, KeyError) as error: return f"host reservation marker unreadable ({type(error).__name__}); kept as reserved" - if since <= now < until: + if now < until: # the runner hook's rule: a marker holds from when it is written until until, since or not return f"host reservation by {held.get('owner', '?')} until {until}" return "" diff --git a/scripts/test-glaeda-disk.py b/scripts/test-glaeda-disk.py index 479e20c0c..15ffc0f05 100644 --- a/scripts/test-glaeda-disk.py +++ b/scripts/test-glaeda-disk.py @@ -6310,6 +6310,16 @@ def test_a_local_marker_that_is_there_but_unreadable_holds_the_host(self) -> Non with self.subTest(text=text): self.local.write_text(text) self.assertIn("marker unreadable", self.reserved({"cmux31"})) + self.local.write_text(json.dumps({"schema": "glaeda-reservation/v1", "since": now + 600, "until": now + 900})) + self.assertIn("host reservation", self.reserved({"cmux31"}), "a future since holds, as the runner hook reads it") + real_read = Path.read_text + + def read_text(path, *args, **kwargs): + if path == self.local: + raise PermissionError("denied") + return real_read(path, *args, **kwargs) + with mock.patch.object(Path, "read_text", read_text): + self.assertIn("marker unreadable (PermissionError)", self.reserved({"cmux31"})) self.local.unlink() self.assertEqual(self.reserved({"cmux31"}), "", "no marker, no hold") From 7fe94c3a7ad675dcbbbaace3dfa6fba8147b2f25 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 26 Oct 2026 16:11:40 -0700 Subject: [PATCH 20/28] glaeda-disk: mirror bundling re-asks the servers, fails closed on read errors, honours locks inside Addresses the third Codex review of #1625: 1. The commits the bundles leave to the servers (their prerequisites from rev-list --boundary, and tips no bundle holds) are asked about again right before the delete; one no server holds any more keeps the tree. 2. The repository search for a mirror item is strict: a directory that cannot be read (EIO, ESTALE) makes it incomplete, and the item stays. 3. A locked linked worktree keeps the item even when its repository is inside it, as in git_loss. Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- scripts/glaeda-disk | 107 +++++++++++++++++++++++------------- scripts/test-glaeda-disk.py | 48 +++++++++++++++- 2 files changed, 116 insertions(+), 39 deletions(-) diff --git a/scripts/glaeda-disk b/scripts/glaeda-disk index f7fb17256..ba258074f 100755 --- a/scripts/glaeda-disk +++ b/scripts/glaeda-disk @@ -4056,9 +4056,10 @@ def looks_like_repository(d: str) -> bool: and os.path.isdir(os.path.join(d, "refs"))) -def checkouts(path: Path, depth: int = 5, budget: int = 20000) -> list[Path] | None: +def checkouts(path: Path, depth: int = 5, budget: int = 20000, strict: bool = False) -> list[Path] | None: """Every checkout (a directory holding .git) or bare repository up to `depth` levels below - path, not searching inside one; None when the search could not finish.""" + path, not searching inside one; None when the search could not finish (with strict, also when any + directory could not be read).""" found: list[Path] = [] frontier = [str(path)] for _ in range(depth + 1): @@ -4081,7 +4082,8 @@ def checkouts(path: Path, depth: int = 5, budget: int = 20000) -> list[Path] | N except PermissionError: return None except OSError: - pass + if strict: + return None frontier = nxt return None if frontier else found @@ -4515,10 +4517,47 @@ def _server_acks(gitdir: Path, remotes: list[str], commits: list[str]) -> set[st return acked -def bundle_unpushed(gitdir: Path, key: str, written: list[Path]) -> str: +def _server_known(gitdir: Path, remotes: list[str], asked: set[str]) -> tuple[str, dict[str, set[str]], list[str]]: + """What the servers hold right now: ("" or why not, refname -> objects they advertise there, the local + commits and tags among those and `asked` they advertise or acknowledge).""" + held: dict[str, set[str]] = {} + for r in remotes: + if (out := _git_dir_run(gitdir, "ls-remote", "--heads", "--tags", r, network=True, timeout=120)) is None: + return f"remote {r} of {gitdir} did not answer", {}, [] + for ln in out.splitlines(): + obj, _, name = ln.partition("\t") + held.setdefault(name.removesuffix("^{}"), set()).add(obj) + candidates = sorted({*asked, *(o for s in held.values() for o in s)}) + have = _git_dir_run(gitdir, "cat-file", "--batch-check=%(objectname) %(objecttype)", + stdin="\n".join(candidates) + "\n") if candidates else "" + if have is None: + return f"unreadable objects in {gitdir}", {}, [] + present = {ln.split()[0] for ln in have.splitlines() if ln.endswith(" commit") or ln.endswith(" tag")} + advertised = {o for s in held.values() for o in s} & present + ask = sorted((present & asked) - advertised) + acked = _server_acks(gitdir, remotes, ask) if len(ask) <= NEGOTIATE_MAX else set() + return "", held, sorted(advertised | (acked & present)) + + +def server_still_holds(gitdir: Path, relied: list[str]) -> str: + """Right before the delete: every commit the bundles left to the servers (their prerequisites and the + tips no bundle holds) is still on a server, asked again.""" + remotes = network_remotes(gitdir) + if remotes is None: + return f"unreadable remotes in {gitdir}" + why, _, known = _server_known(gitdir, remotes, set(relied)) + if why: + return why + out = _git_dir_run(gitdir, "rev-list", "--stdin", stdin="\n".join([*relied, "--not", *known]) + "\n", + timeout=600) if relied else "" + return "" if out == "" else f"a commit left to the servers is no longer on one ({gitdir})" + + +def bundle_unpushed(gitdir: Path, key: str, written: list[Path], relied: list[str] | None = None) -> str: """Bundle what deleting this repository loses: commits no remote server confirms (what ls-remote advertises and what `fetch --negotiate-only` acknowledges; local remote-tracking refs are only candidates to ask - about), and every branch or tag the servers do not have at that commit. "" or why it failed.""" + about), and every branch or tag the servers do not have at that commit. The commits left to the servers + go to relied, for server_still_holds. "" or why it failed.""" tips = local_tips(gitdir) for wt in [p for p in (gitdir / "worktrees").glob("*") if p.is_dir()] if tips is not None else []: own = local_tips(wt, worktree_only=True) @@ -4535,28 +4574,17 @@ def bundle_unpushed(gitdir: Path, key: str, written: list[Path]) -> str: tracking.add(peeled or obj) else: local[name] = (obj, peeled or obj) - held: dict[str, set[str]] = {} # refname on a server -> objects it advertises there - for r in remotes: - if (out := _git_dir_run(gitdir, "ls-remote", "--heads", "--tags", r, network=True, timeout=120)) is None: - return f"remote {r} of {gitdir} did not answer" - for ln in out.splitlines(): - obj, _, name = ln.partition("\t") - held.setdefault(name.removesuffix("^{}"), set()).add(obj) - candidates = sorted({*tips, *tracking, *(o for s in held.values() for o in s)}) - have = _git_dir_run(gitdir, "cat-file", "--batch-check=%(objectname) %(objecttype)", - stdin="\n".join(candidates) + "\n") if candidates else "" - if have is None: - return f"unreadable objects in {gitdir}" - present = {ln.split()[0] for ln in have.splitlines() if ln.endswith(" commit") or ln.endswith(" tag")} - advertised = {o for s in held.values() for o in s} & present - ask = sorted((present & ({*tips} | tracking)) - advertised) - acked = _server_acks(gitdir, remotes, ask) if len(ask) <= NEGOTIATE_MAX else set() - known = sorted(advertised | (acked & present)) - out = _git_dir_run(gitdir, "rev-list", "--stdin", stdin="\n".join([*tips, "--not", *known]) + "\n", - timeout=600) + why, held, known = _server_known(gitdir, remotes, {*tips, *tracking}) + if why: + return why + # --boundary: the server-held commits the unpushed history stops at, which its bundle requires + out = _git_dir_run(gitdir, "rev-list", "--boundary", "--stdin", + stdin="\n".join([*tips, "--not", *known]) + "\n", timeout=600) if out is None: return f"unreadable history in {gitdir}" - unpushed = set(out.split()) + unpushed = {c for c in out.split() if not c.startswith("-")} + if relied is not None: + relied += sorted({c[1:] for c in out.split() if c.startswith("-")} | ({*tips} - unpushed)) # the refs to keep: every local one on an unpushed commit, and every branch or tag the servers lack there keep = {n for n, (_, c) in local.items() if c in unpushed} only_here = sorted(n for n, (o, _) in local.items() if n.startswith(("refs/heads/", "refs/tags/")) @@ -4596,7 +4624,7 @@ def _ignored_checkouts(root: Path) -> list[Path] | None: found: list[Path] = [] for entry in r.stdout.split("\0"): if entry.startswith("!! ") and entry.endswith("/") and Path(entry[3:-1]).name not in REPRODUCIBLE_DIRS: - if (inner := checkouts(root / entry[3:-1], NESTED_DEPTH, NESTED_BUDGET)) is None: + if (inner := checkouts(root / entry[3:-1], NESTED_DEPTH, NESTED_BUDGET, strict=True)) is None: return None found += inner return found @@ -4638,9 +4666,9 @@ def mirror_repos(path: Path) -> tuple[str, dict[str, Path], list[Path]]: repositories, a linked worktree's repository inside it, checkouts in ignored directories, submodule stores and initialized submodules, recursively), and its work trees, each checked clean right now. ("" or why it must stay, real git dir -> git dir, work trees).""" - queue = nested_checkouts(path) + queue = checkouts(path, NESTED_DEPTH, NESTED_BUDGET, strict=True) # an unreadable directory may hide one if queue is None: - return "too deep or large to find every checkout", {}, [] + return "too deep, large or unreadable to find every checkout", {}, [] queue, inside = list(queue), os.path.realpath(path) + "/" gitdirs: dict[str, Path] = {} trees: list[Path] = [] @@ -4676,11 +4704,11 @@ def mirror_repos(path: Path) -> tuple[str, dict[str, Path], list[Path]]: common = git_common_dir(root) if not common: return f"{where}: unreadable worktree", {}, [] + if os.path.lexists(os.path.join(own.strip(), "locked")): # as git_loss: wherever its repository is + return f"{where}: locked worktree", {}, [] if (os.path.realpath(common) + "/").startswith(inside): gitdirs[os.path.realpath(common)] = Path(common) continue - if os.path.lexists(os.path.join(own.strip(), "locked")): - return f"{where}: locked worktree", {}, [] tips = local_tips(Path(own.strip()), worktree_only=True) left = None if tips is None else _git_dir_run( Path(common), "rev-list", "-n1", *tips, "--not", "--branches", "--tags", "--remotes") if tips else "" @@ -4700,26 +4728,27 @@ def mirror_repos(path: Path) -> tuple[str, dict[str, Path], list[Path]]: return "", gitdirs, sorted(trees) -def archive_mirror(path: Path) -> tuple[str, list[Path], tuple[list[Path], list[Path], list[str | None]]]: +def archive_mirror(path: Path) -> tuple[str, list[Path], tuple]: """Before a mirror item goes: bundle_unpushed for every repository mirror_repos finds. A tree with uncommitted work, or whose repositories cannot all be found or bundled, is kept. ("" or why it must stay, the bundles written, what mirror_moved rechecks).""" - none: tuple[list[Path], list[Path], list[str | None]] = ([], [], []) + none: tuple = ([], [], [], {}) why, gitdirs, trees = mirror_repos(path) if why: return why, [], none snap = mirror_git_snapshot(list(gitdirs.values()), trees) bundles: list[Path] = [] + relied: dict[Path, list[str]] = {} for key, g in sorted(gitdirs.items()): - if (why := bundle_unpushed(g, key, bundles)): + if (why := bundle_unpushed(g, key, bundles, relied.setdefault(g, []))): return why, bundles, none - return "", bundles, (list(gitdirs.values()), trees, snap) + return "", bundles, (list(gitdirs.values()), trees, snap, relied) -def mirror_moved(fam: Family, path: Path, recheck: tuple[list[Path], list[Path], list[str | None]]) -> str: +def mirror_moved(fam: Family, path: Path, recheck: tuple) -> str: """Right before a bundled mirror goes, everything the bundling took time over is read again: process evidence, the family's owner and artifact references, and its repositories, found afresh (the same set, - every work tree clean) and as at bundle time.""" + every work tree clean) and as at bundle time, and the servers, asked again for what the bundles left to them.""" try: opened, cmds = process_evidence() except NoEvidence as e: @@ -4731,14 +4760,16 @@ def mirror_moved(fam: Family, path: Path, recheck: tuple[list[Path], list[Path], return "its owner loaded while its git work was bundled" if fam.reference_root is not None and artifact_referenced(path, fresh_artifact_references(fam.reference_root)): return "referenced while its git work was bundled" - gitdirs, trees, snap = recheck - _NESTED_WALKS.pop(str(path), None) + gitdirs, trees, snap, relied = recheck why, now, now_trees = mirror_repos(path) if why or {os.path.realpath(g) for g in now.values()} != {os.path.realpath(g) for g in gitdirs} \ or now_trees != trees: return f"its repositories changed while they were bundled{': ' + why if why else ''}" if None in snap or mirror_git_snapshot(gitdirs, trees) != snap: return "its git refs or work trees changed while they were bundled" + for g, commits in relied.items(): + if (why := server_still_holds(g, commits)): + return why return "" diff --git a/scripts/test-glaeda-disk.py b/scripts/test-glaeda-disk.py index a6529be96..633f004d2 100644 --- a/scripts/test-glaeda-disk.py +++ b/scripts/test-glaeda-disk.py @@ -1518,7 +1518,7 @@ def git_state(f, path): # a cargo target dir too large to search mock.patch.object(gd, "process_evidence", return_value=late), \ mock.patch.object(gd, "command_lines", return_value=late[1]), \ mock.patch.object(gd, "HOME", self.root / "home"), \ - mock.patch.object(gd, "archive_mirror", return_value=("", [], ([], [], []))), \ + mock.patch.object(gd, "archive_mirror", return_value=("", [], ([], [], [], {}))), \ mock.patch.object(gd, "mirror_moved", return_value=""): # bundling: the cmux-job tests gd.apply(reclaim, {fam.id: fam}, receipt, None, 24) self.assertEqual(sorted(p.name for p in cache.iterdir()), @@ -5900,6 +5900,52 @@ def test_a_locked_linked_worktree_keeps_the_mirror(self) -> None: self.assertTrue(any("locked worktree" in r for r in rec["reasons"])) self.assertTrue((unit / "slot-1/.git").is_file()) + def test_a_branch_the_server_drops_after_bundling_keeps_the_mirror(self) -> None: + unit, clone, env = self.mirror_unit() + srv = self.home / "srv" + self.git(srv, "checkout", "-q", "-b", "feat", env=env) + self.git(srv, "commit", "-q", "--allow-empty", "-m", "feat", env=env) + self.git(srv, "checkout", "-q", "main", env=env) + self.git(clone, "fetch", "-q", "origin", env=env) + self.git(clone, "branch", "mine", "origin/feat", env=env) # on the server now: no bundle needed + real = gd.archive_mirror + + def racing(path): + out = real(path) + self.git(srv, "branch", "-q", "-D", "feat", env=env) + self.git(srv, "reflog", "expire", "--expire=now", "--all", env=env) + self.git(srv, "gc", "-q", "--prune=now", env=env) + return out + with mock.patch.object(gd, "archive_mirror", racing): + rec, _ = self.apply_mirror(unit) + self.assertEqual(rec["outcome"], "changed:in-use") + self.assertTrue(any("no longer on one" in r for r in rec["reasons"]), rec["reasons"]) + self.assertTrue((clone / ".git").is_dir()) + + def test_a_directory_read_error_while_finding_repositories_keeps_the_mirror(self) -> None: + unit, _, _ = self.mirror_unit() + hidden = unit / "slot-2/deep" + hidden.mkdir(parents=True) + real = os.scandir + + def failing(p): + if os.fspath(p) == os.fspath(unit / "slot-2"): + raise OSError(errno.EIO, "I/O error") + return real(p) + with mock.patch.object(gd.os, "scandir", failing): + why, bundles, _ = gd.archive_mirror(unit) + self.assertIn("unreadable", why) + self.assertEqual(bundles, []) + self.assertEqual(gd.checkouts(unit, strict=False) is None, False) + + def test_a_locked_worktree_of_a_repository_inside_the_mirror_keeps_it(self) -> None: + unit, clone, env = self.mirror_unit() + self.git(clone, "worktree", "add", "-q", "--detach", "--lock", str(unit / "slot-2"), env=env) + rec, _ = self.apply_mirror(unit) + self.assertEqual(rec["outcome"], "blocked:bundle-failed") + self.assertTrue(any("locked worktree" in r for r in rec["reasons"]), rec["reasons"]) + self.assertTrue((unit / "slot-2/.git").is_file()) + def test_only_manifest_listed_bundles_at_their_hash_are_pruned_after_the_evidence_gate(self) -> None: archive = self.home / "archive/glaeda-disk" archive.mkdir(parents=True) From 9e0e678abb74bfa9c224f6f136608467638731f8 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 26 Oct 2026 16:23:56 -0700 Subject: [PATCH 21/28] glaeda-disk: mirror recheck asks the servers first, symlinked module stores keep, empty repos go Addresses the fourth Codex review of #1625: 1. mirror_moved asks the servers first and rereads the repositories and their snapshot last, so a local commit made while the servers were asked keeps the tree. 2. A symlinked modules/ store, or a symlinked directory inside one, is unreadable, as in submodule_loss. 3. An unborn repository (no refs, no reflog, HEAD naming a branch with no commit) needs no bundle; the snapshot reads HEAD as a file, so it is comparable too. Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- scripts/glaeda-disk | 20 ++++++++++++++++---- scripts/test-glaeda-disk.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/scripts/glaeda-disk b/scripts/glaeda-disk index ba258074f..f1ab36fdd 100755 --- a/scripts/glaeda-disk +++ b/scripts/glaeda-disk @@ -4558,6 +4558,11 @@ def bundle_unpushed(gitdir: Path, key: str, written: list[Path], relied: list[st and what `fetch --negotiate-only` acknowledges; local remote-tracking refs are only candidates to ask about), and every branch or tag the servers do not have at that commit. The commits left to the servers go to relied, for server_still_holds. "" or why it failed.""" + if _git_dir_run(gitdir, "for-each-ref", "--count=1") == "" and _git_dir_run( + gitdir, "rev-parse", "-q", "--verify", "HEAD") is None and _git_dir_run( + gitdir, "symbolic-ref", "-q", "HEAD") is not None and not any( + f.is_file() for f in (gitdir / "logs").rglob("*")): + return "" # unborn: no ref, no reflog, HEAD names a branch with no commit yet; nothing to keep tips = local_tips(gitdir) for wt in [p for p in (gitdir / "worktrees").glob("*") if p.is_dir()] if tips is not None else []: own = local_tips(wt, worktree_only=True) @@ -4649,7 +4654,10 @@ def mirror_git_snapshot(gitdirs: list[Path], trees: list[Path]) -> list[str | No for g in gitdirs: snap.append(_git_dir_run(g, "for-each-ref", "--format=%(refname) %(objectname)")) for d in [g, *(p for p in (g / "worktrees").glob("*") if p.is_dir())]: - snap.append(_git_dir_run(d, "rev-parse", "HEAD")) + try: + snap.append((d / "HEAD").read_text()) # a symbolic ref or a commit; an unborn HEAD reads too + except OSError: + snap.append(None) logs = d / "logs" try: snap.append(" ".join(f"{f}:{f.stat().st_size}" for f in sorted(logs.rglob("*")) if f.is_file()) @@ -4716,7 +4724,11 @@ def mirror_repos(path: Path) -> tuple[str, dict[str, Path], list[Path]]: return f"{where}: commits only this worktree's reflog holds", {}, [] unreadable: list[str] = [] for g in list(gitdirs.values()): # submodule stores: modules/, nested ones too + if (g / "modules").is_symlink(): + unreadable.append(os.fspath(g / "modules")) for dirpath, dirnames, _ in os.walk(g / "modules", onerror=lambda e: unreadable.append(str(e.filename))): + # os.walk neither enters nor reports a symlinked directory: as submodule_loss, it is unreadable + unreadable += [os.path.join(dirpath, d) for d in dirnames if os.path.islink(os.path.join(dirpath, d))] if looks_like_repository(dirpath): gitdirs[os.path.realpath(dirpath)] = Path(dirpath) dirnames[:] = ["modules"] if "modules" in dirnames else [] @@ -4761,15 +4773,15 @@ def mirror_moved(fam: Family, path: Path, recheck: tuple) -> str: if fam.reference_root is not None and artifact_referenced(path, fresh_artifact_references(fam.reference_root)): return "referenced while its git work was bundled" gitdirs, trees, snap, relied = recheck + for g, commits in relied.items(): # the network first: the local checks below are the last word + if (why := server_still_holds(g, commits)): + return why why, now, now_trees = mirror_repos(path) if why or {os.path.realpath(g) for g in now.values()} != {os.path.realpath(g) for g in gitdirs} \ or now_trees != trees: return f"its repositories changed while they were bundled{': ' + why if why else ''}" if None in snap or mirror_git_snapshot(gitdirs, trees) != snap: return "its git refs or work trees changed while they were bundled" - for g, commits in relied.items(): - if (why := server_still_holds(g, commits)): - return why return "" diff --git a/scripts/test-glaeda-disk.py b/scripts/test-glaeda-disk.py index 633f004d2..039dfb93b 100644 --- a/scripts/test-glaeda-disk.py +++ b/scripts/test-glaeda-disk.py @@ -5946,6 +5946,38 @@ def test_a_locked_worktree_of_a_repository_inside_the_mirror_keeps_it(self) -> N self.assertTrue(any("locked worktree" in r for r in rec["reasons"]), rec["reasons"]) self.assertTrue((unit / "slot-2/.git").is_file()) + def test_a_commit_made_while_the_servers_are_asked_keeps_the_mirror(self) -> None: + unit, clone, env = self.mirror_unit() + real = gd.server_still_holds + + def slow(g, commits): + self.git(clone, "commit", "-q", "--allow-empty", "-m", "made meanwhile", env=env) + return real(g, commits) + with mock.patch.object(gd, "server_still_holds", slow): + rec, _ = self.apply_mirror(unit) + self.assertEqual(rec["outcome"], "changed:in-use") + self.assertTrue((clone / ".git").is_dir()) + + def test_a_symlinked_submodule_store_keeps_the_mirror(self) -> None: + unit, clone, _ = self.mirror_unit() + (unit / "store/sub").mkdir(parents=True) + (clone / ".git/modules").symlink_to(unit / "store") + rec, _ = self.apply_mirror(unit) + self.assertEqual(rec["outcome"], "blocked:bundle-failed") + self.assertTrue(any("unreadable submodule store" in r for r in rec["reasons"]), rec["reasons"]) + (clone / ".git/modules").unlink() + (clone / ".git/modules/a").mkdir(parents=True) + (clone / ".git/modules/a/b").symlink_to(unit / "store") + why, _, _ = gd.archive_mirror(unit) + self.assertIn("unreadable submodule store", why) + + def test_an_empty_repository_goes_without_a_bundle(self) -> None: + unit = self.home / ".cache/cmux-job/reload-cloud/cmux-base" + subprocess.run(["git", "init", "-q", "-b", "main", str(unit / "slot-1")], check=True) + rec, _ = self.apply_mirror(unit) + self.assertEqual(rec["outcome"], "reclaimed", rec["reasons"]) + self.assertFalse((self.home / "archive/glaeda-disk").exists()) + def test_only_manifest_listed_bundles_at_their_hash_are_pruned_after_the_evidence_gate(self) -> None: archive = self.home / "archive/glaeda-disk" archive.mkdir(parents=True) From e5fbb865ad44cd02ae664bba5ba5178818428e58 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 26 Oct 2026 16:46:57 -0700 Subject: [PATCH 22/28] glaeda-disk: mirror recheck ends on process and owner checks; unborn and worktree reads fail closed Addresses the fifth Codex review of #1625: 1. mirror_moved now asks the servers, then rereads git, then the owner, artifact references and fresh process evidence last, so a job that starts during the slow steps keeps the tree. 2. The unborn-repository fast path applies only when the repository has no linked worktrees. 3. Linked worktree records are read strictly: an unreadable worktrees/ or record keeps the item. Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- scripts/glaeda-disk | 46 +++++++++++++++++++++++++------------ scripts/test-glaeda-disk.py | 43 ++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 15 deletions(-) diff --git a/scripts/glaeda-disk b/scripts/glaeda-disk index 64dc08c3c..4d2c96d6d 100755 --- a/scripts/glaeda-disk +++ b/scripts/glaeda-disk @@ -4566,8 +4566,8 @@ def bundle_unpushed(gitdir: Path, key: str, written: list[Path], relied: list[st if _git_dir_run(gitdir, "for-each-ref", "--count=1") == "" and _git_dir_run( gitdir, "rev-parse", "-q", "--verify", "HEAD") is None and _git_dir_run( gitdir, "symbolic-ref", "-q", "HEAD") is not None and not any( - f.is_file() for f in (gitdir / "logs").rglob("*")): - return "" # unborn: no ref, no reflog, HEAD names a branch with no commit yet; nothing to keep + f.is_file() for f in (gitdir / "logs").rglob("*")) and not os.path.lexists(gitdir / "worktrees"): + return "" # unborn: no ref, reflog or linked worktree, HEAD a branch with no commit yet; nothing to keep tips = local_tips(gitdir) for wt in [p for p in (gitdir / "worktrees").glob("*") if p.is_dir()] if tips is not None else []: own = local_tips(wt, worktree_only=True) @@ -4674,6 +4674,20 @@ def mirror_git_snapshot(gitdirs: list[Path], trees: list[Path]) -> list[str | No return snap +def _linked_trees_strict(gitdir: Path) -> list[Path] | None: + """linked_work_trees, but None when /worktrees or any record in it cannot be read.""" + wts = gitdir / "worktrees" + if not os.path.lexists(wts): + return [] + try: + if wts.is_symlink() or not wts.is_dir(): + return None + with os.scandir(wts) as it: + return [Path((Path(e.path) / "gitdir").read_text().strip()).parent for e in it] + except (OSError, ValueError): + return None + + def mirror_repos(path: Path) -> tuple[str, dict[str, Path], list[Path]]: """Every repository a mirror item takes along, found as _nested_loss finds them (checkouts, bare repositories, a linked worktree's repository inside it, checkouts in ignored directories, submodule stores @@ -4740,7 +4754,9 @@ def mirror_repos(path: Path) -> tuple[str, dict[str, Path], list[Path]]: if [u for u in unreadable if os.path.lexists(u)]: # a missing modules/ is no submodule store return f"unreadable submodule store {unreadable[0]}", {}, [] for key, g in gitdirs.items(): - if any(t.exists() and not (os.path.realpath(t) + "/").startswith(inside) for t in linked_work_trees(g)): + if (linked := _linked_trees_strict(g)) is None: + return f"{key}: unreadable linked worktree records", {}, [] + if any(t.exists() and not (os.path.realpath(t) + "/").startswith(inside) for t in linked): return f"{key}: a worktree outside this item uses it", {}, [] return "", gitdirs, sorted(trees) @@ -4766,19 +4782,8 @@ def mirror_moved(fam: Family, path: Path, recheck: tuple) -> str: """Right before a bundled mirror goes, everything the bundling took time over is read again: process evidence, the family's owner and artifact references, and its repositories, found afresh (the same set, every work tree clean) and as at bundle time, and the servers, asked again for what the bundles left to them.""" - try: - opened, cmds = process_evidence() - except NoEvidence as e: - return f"no process evidence: {e}" - if in_cwd(os.fspath(path), open_index(opened)) or named_by(os.fspath(path), cmds) or holds_socket(path) \ - or job_busy(fam, path): - return "in use while its git work was bundled" - if fam.retired_only and not owner_retired(fam): - return "its owner loaded while its git work was bundled" - if fam.reference_root is not None and artifact_referenced(path, fresh_artifact_references(fam.reference_root)): - return "referenced while its git work was bundled" gitdirs, trees, snap, relied = recheck - for g, commits in relied.items(): # the network first: the local checks below are the last word + for g, commits in relied.items(): # the slow network first, then git, then the quick process checks last if (why := server_still_holds(g, commits)): return why why, now, now_trees = mirror_repos(path) @@ -4787,6 +4792,17 @@ def mirror_moved(fam: Family, path: Path, recheck: tuple) -> str: return f"its repositories changed while they were bundled{': ' + why if why else ''}" if None in snap or mirror_git_snapshot(gitdirs, trees) != snap: return "its git refs or work trees changed while they were bundled" + if fam.retired_only and not owner_retired(fam): + return "its owner loaded while its git work was bundled" + if fam.reference_root is not None and artifact_referenced(path, fresh_artifact_references(fam.reference_root)): + return "referenced while its git work was bundled" + try: + opened, cmds = process_evidence() + except NoEvidence as e: + return f"no process evidence: {e}" + if in_cwd(os.fspath(path), open_index(opened)) or named_by(os.fspath(path), cmds) or holds_socket(path) \ + or job_busy(fam, path): + return "in use while its git work was bundled" return "" diff --git a/scripts/test-glaeda-disk.py b/scripts/test-glaeda-disk.py index 165da5d9f..48207f0ae 100644 --- a/scripts/test-glaeda-disk.py +++ b/scripts/test-glaeda-disk.py @@ -5997,6 +5997,49 @@ def test_an_empty_repository_goes_without_a_bundle(self) -> None: self.assertEqual(rec["outcome"], "reclaimed", rec["reasons"]) self.assertFalse((self.home / "archive/glaeda-disk").exists()) + def test_a_job_that_starts_while_the_servers_are_asked_keeps_the_mirror(self) -> None: + unit, clone, _ = self.mirror_unit() + busy = [] + real = gd.server_still_holds + + def slow(g, commits): + busy.append(1) # a job takes the unit during the network wait + return real(g, commits) + with mock.patch.object(gd, "server_still_holds", slow), \ + mock.patch.object(gd, "job_busy", lambda fam, p: bool(busy)): + rec, _ = self.apply_mirror(unit) + self.assertEqual(rec["outcome"], "changed:in-use") + self.assertIn("in use while its git work was bundled", rec["reasons"]) + self.assertTrue((clone / ".git").is_dir()) + + def test_an_unborn_repositorys_linked_worktree_commit_is_never_lost(self) -> None: + unit, clone, env = self.mirror_unit() + self.git(clone, "worktree", "add", "-q", "--detach", str(unit / "slot-2"), env=env) + self.git(unit / "slot-2", "commit", "-q", "--allow-empty", "-m", "worktree only", env=env) + only = self.git(unit / "slot-2", "rev-parse", "HEAD") + for ref in self.git(clone, "for-each-ref", "--format=%(refname)").split(): + self.git(clone, "update-ref", "-d", "--no-deref", ref, env=env) + shutil.rmtree(clone / ".git/logs") # the common repository now reads as unborn + rec, _ = self.apply_mirror(unit) + if rec["outcome"] == "reclaimed": + self.assertIn(only, {c for _, c in self.bundled_pairs()}) + else: + self.assertTrue((unit / "slot-2/.git").is_file()) + + def test_unreadable_linked_worktree_records_keep_the_mirror(self) -> None: + unit, clone, env = self.mirror_unit() + self.git(clone, "worktree", "add", "-q", "--detach", str(self.home / "outside"), env=env) + record = next((clone / ".git/worktrees").iterdir()) / "gitdir" + os.chmod(record, 0) + try: + why, bundles, _ = gd.archive_mirror(unit) + finally: + with contextlib.suppress(OSError): + os.chmod(record, 0o644) + self.assertTrue(record.exists(), "nothing prunes the record") + self.assertIn("unreadable linked worktree records", why) + self.assertEqual(bundles, []) + def test_only_manifest_listed_bundles_at_their_hash_are_pruned_after_the_evidence_gate(self) -> None: archive = self.home / "archive/glaeda-disk" archive.mkdir(parents=True) From 6da80b845340a8e6b5e4bf156fda611a088dead0 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 26 Oct 2026 17:03:19 -0700 Subject: [PATCH 23/28] glaeda-disk: mirror bundling reads reflogs strictly and checks linked worktrees inside the item Addresses the sixth Codex review of #1625: 1. Reflog directories are walked with files_strict (os.walk with onerror), so an unreadable one makes local_tips, the unborn check and the snapshot fail closed instead of skipping it. 2. A linked worktree inside the item (even one under ignored output such as target/) is walked like any work tree: it must be clean and unlocked. Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- scripts/glaeda-disk | 33 ++++++++++++++++++++++++--------- scripts/test-glaeda-disk.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 9 deletions(-) diff --git a/scripts/glaeda-disk b/scripts/glaeda-disk index 4d2c96d6d..3e31c6c75 100755 --- a/scripts/glaeda-disk +++ b/scripts/glaeda-disk @@ -4301,6 +4301,16 @@ def _git_dir_run(gitdir: Path, *args: str, timeout: int = 60, ZERO = re.compile(r"0{40}(0{24})?$") +def files_strict(d: Path) -> list[Path] | None: + """Every file below d (none when it is absent); None when any directory below it cannot be read, which + Path.rglob would skip without a word.""" + if not os.path.lexists(d): + return [] + errors: list[OSError] = [] + found = [Path(dp) / f for dp, _, fs in os.walk(d, onerror=errors.append) for f in fs] + return None if errors or d.is_symlink() else found + + # refs a linked worktree keeps for itself; they are deleted with it WORKTREE_REFS = ("refs/bisect", "refs/worktree", "refs/rewritten") @@ -4322,8 +4332,10 @@ def local_tips(gitdir: Path, worktree_only: bool = False) -> list[str] | None: if not name.startswith("refs/remotes/"): tips.add(peeled or obj) logs = gitdir / "logs" - if logs.is_dir(): - for f in logs.rglob("*"): + if (files := files_strict(logs)) is None: + return None + if files: + for f in files: if f.is_file() and not f.relative_to(logs).parts[:2] == ("refs", "remotes"): try: lines = f.read_text(errors="replace").splitlines() @@ -4565,8 +4577,8 @@ def bundle_unpushed(gitdir: Path, key: str, written: list[Path], relied: list[st go to relied, for server_still_holds. "" or why it failed.""" if _git_dir_run(gitdir, "for-each-ref", "--count=1") == "" and _git_dir_run( gitdir, "rev-parse", "-q", "--verify", "HEAD") is None and _git_dir_run( - gitdir, "symbolic-ref", "-q", "HEAD") is not None and not any( - f.is_file() for f in (gitdir / "logs").rglob("*")) and not os.path.lexists(gitdir / "worktrees"): + gitdir, "symbolic-ref", "-q", "HEAD") is not None and files_strict(gitdir / "logs") == [] \ + and not os.path.lexists(gitdir / "worktrees"): return "" # unborn: no ref, reflog or linked worktree, HEAD a branch with no commit yet; nothing to keep tips = local_tips(gitdir) for wt in [p for p in (gitdir / "worktrees").glob("*") if p.is_dir()] if tips is not None else []: @@ -4663,10 +4675,9 @@ def mirror_git_snapshot(gitdirs: list[Path], trees: list[Path]) -> list[str | No snap.append((d / "HEAD").read_text()) # a symbolic ref or a commit; an unborn HEAD reads too except OSError: snap.append(None) - logs = d / "logs" try: - snap.append(" ".join(f"{f}:{f.stat().st_size}" for f in sorted(logs.rglob("*")) if f.is_file()) - if logs.is_dir() else "") + files = files_strict(d / "logs") + snap.append(None if files is None else " ".join(f"{f}:{f.stat().st_size}" for f in sorted(files))) except OSError: snap.append(None) # unreadable: never matches, so the tree stays snap += [_git_dir_run(t / ".git", "--work-tree", str(t), "status", "--porcelain", "--untracked-files=all") @@ -4688,7 +4699,7 @@ def _linked_trees_strict(gitdir: Path) -> list[Path] | None: return None -def mirror_repos(path: Path) -> tuple[str, dict[str, Path], list[Path]]: +def mirror_repos(path: Path, linked_inside: tuple[Path, ...] = ()) -> tuple[str, dict[str, Path], list[Path]]: """Every repository a mirror item takes along, found as _nested_loss finds them (checkouts, bare repositories, a linked worktree's repository inside it, checkouts in ignored directories, submodule stores and initialized submodules, recursively), and its work trees, each checked clean right now. @@ -4696,7 +4707,7 @@ def mirror_repos(path: Path) -> tuple[str, dict[str, Path], list[Path]]: queue = checkouts(path, NESTED_DEPTH, NESTED_BUDGET, strict=True) # an unreadable directory may hide one if queue is None: return "too deep, large or unreadable to find every checkout", {}, [] - queue, inside = list(queue), os.path.realpath(path) + "/" + queue, inside = [*queue, *linked_inside], os.path.realpath(path) + "/" gitdirs: dict[str, Path] = {} trees: list[Path] = [] seen: set[str] = set() @@ -4753,11 +4764,15 @@ def mirror_repos(path: Path) -> tuple[str, dict[str, Path], list[Path]]: dirnames[:] = ["modules"] if "modules" in dirnames else [] if [u for u in unreadable if os.path.lexists(u)]: # a missing modules/ is no submodule store return f"unreadable submodule store {unreadable[0]}", {}, [] + found: list[Path] = [] for key, g in gitdirs.items(): if (linked := _linked_trees_strict(g)) is None: return f"{key}: unreadable linked worktree records", {}, [] if any(t.exists() and not (os.path.realpath(t) + "/").startswith(inside) for t in linked): return f"{key}: a worktree outside this item uses it", {}, [] + found += [t for t in linked if os.path.lexists(t) and os.path.realpath(t) not in seen] + if found: # a linked worktree inside the item, even in ignored output: checked clean and unlocked too + return mirror_repos(path, (*linked_inside, *found)) return "", gitdirs, sorted(trees) diff --git a/scripts/test-glaeda-disk.py b/scripts/test-glaeda-disk.py index 48207f0ae..fa0602d3b 100644 --- a/scripts/test-glaeda-disk.py +++ b/scripts/test-glaeda-disk.py @@ -6040,6 +6040,35 @@ def test_unreadable_linked_worktree_records_keep_the_mirror(self) -> None: self.assertIn("unreadable linked worktree records", why) self.assertEqual(bundles, []) + def test_an_unreadable_reflog_directory_keeps_the_mirror(self) -> None: + unit, clone, _ = self.mirror_unit() + hidden = clone / ".git/logs/refs/heads" + os.chmod(hidden, 0) + try: + self.assertIsNone(gd.files_strict(clone / ".git/logs")) + self.assertIsNone(gd.local_tips(clone / ".git")) + why, bundles, _ = gd.archive_mirror(unit) + finally: + os.chmod(hidden, 0o755) + self.assertTrue(why) + self.assertEqual(bundles, []) + + def test_a_dirty_or_locked_worktree_in_ignored_output_keeps_the_mirror(self) -> None: + unit, clone, env = self.mirror_unit() + with (clone / ".git/info/exclude").open("a") as f: + f.write("target/\n") + wt = clone / "target/wt" + self.git(clone, "worktree", "add", "-q", "--detach", str(wt), env=env) + (wt / "edit.txt").write_text("uncommitted") + why, _, _ = gd.archive_mirror(unit) + self.assertIn("uncommitted", why) + (wt / "edit.txt").unlink() + self.git(clone, "worktree", "lock", str(wt), env=env) + why, _, _ = gd.archive_mirror(unit) + self.assertIn("locked worktree", why) + self.git(clone, "worktree", "unlock", str(wt), env=env) + self.assertEqual(gd.archive_mirror(unit)[0], "") + def test_only_manifest_listed_bundles_at_their_hash_are_pruned_after_the_evidence_gate(self) -> None: archive = self.home / "archive/glaeda-disk" archive.mkdir(parents=True) From f94fa640bd1697c198d17db3a24d136934c20990 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 26 Oct 2026 17:18:12 -0700 Subject: [PATCH 24/28] glaeda-disk: mirror bundling treats symlinked reflog dirs and unreadable linked worktrees as unreadable Addresses the seventh Codex review of #1625: 1. files_strict returns None when a directory below the root is a symlink, which os.walk would not enter, so a symlinked logs/refs/heads cannot hide a reflog-only commit. 2. Linked worktrees are probed with os.stat: only FileNotFoundError means absent; EACCES or EIO keeps the item instead of skipping the outside-worktree check. Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- scripts/glaeda-disk | 19 ++++++++++++++++--- scripts/test-glaeda-disk.py | 22 ++++++++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/scripts/glaeda-disk b/scripts/glaeda-disk index e17532f82..2bfb0ca18 100755 --- a/scripts/glaeda-disk +++ b/scripts/glaeda-disk @@ -4308,7 +4308,11 @@ def files_strict(d: Path) -> list[Path] | None: if not os.path.lexists(d): return [] errors: list[OSError] = [] - found = [Path(dp) / f for dp, _, fs in os.walk(d, onerror=errors.append) for f in fs] + found: list[Path] = [] + for dp, dirs, fs in os.walk(d, onerror=errors.append): + if any(os.path.islink(os.path.join(dp, x)) for x in dirs): # os.walk would not enter it: unread + return None + found += [Path(dp) / f for f in fs] return None if errors or d.is_symlink() else found @@ -4769,9 +4773,18 @@ def mirror_repos(path: Path, linked_inside: tuple[Path, ...] = ()) -> tuple[str, for key, g in gitdirs.items(): if (linked := _linked_trees_strict(g)) is None: return f"{key}: unreadable linked worktree records", {}, [] - if any(t.exists() and not (os.path.realpath(t) + "/").startswith(inside) for t in linked): + present: list[Path] = [] + for t in linked: + try: # os.stat, not exists(): a tree we cannot read (EACCES, EIO) is not an absent one + os.stat(t) + present.append(t) + except FileNotFoundError: + continue + except OSError: + return f"{key}: unreadable linked worktree {t}", {}, [] + if any(not (os.path.realpath(t) + "/").startswith(inside) for t in present): return f"{key}: a worktree outside this item uses it", {}, [] - found += [t for t in linked if os.path.lexists(t) and os.path.realpath(t) not in seen] + found += [t for t in present if os.path.realpath(t) not in seen] if found: # a linked worktree inside the item, even in ignored output: checked clean and unlocked too return mirror_repos(path, (*linked_inside, *found)) return "", gitdirs, sorted(trees) diff --git a/scripts/test-glaeda-disk.py b/scripts/test-glaeda-disk.py index 48766c0f9..2b7b2c09f 100644 --- a/scripts/test-glaeda-disk.py +++ b/scripts/test-glaeda-disk.py @@ -6069,6 +6069,28 @@ def test_a_dirty_or_locked_worktree_in_ignored_output_keeps_the_mirror(self) -> self.git(clone, "worktree", "unlock", str(wt), env=env) self.assertEqual(gd.archive_mirror(unit)[0], "") + def test_a_symlinked_reflog_directory_keeps_the_mirror(self) -> None: + unit, clone, _ = self.mirror_unit() + heads = clone / ".git/logs/refs/heads" + shutil.move(os.fspath(heads), os.fspath(self.home / "heads-elsewhere")) + heads.symlink_to(self.home / "heads-elsewhere") + self.assertIsNone(gd.files_strict(clone / ".git/logs")) + self.assertIsNone(gd.local_tips(clone / ".git")) + self.assertTrue(gd.archive_mirror(unit)[0]) + + def test_an_unreadable_outside_linked_worktree_keeps_the_mirror(self) -> None: + unit, clone, env = self.mirror_unit() + parent = self.home / "locked-away" + parent.mkdir() + self.git(clone, "worktree", "add", "-q", "--detach", str(parent / "wt"), env=env) + os.chmod(parent, 0) + try: + why, bundles, _ = gd.archive_mirror(unit) + finally: + os.chmod(parent, 0o755) + self.assertIn("unreadable linked worktree", why) + self.assertEqual(bundles, []) + def test_only_manifest_listed_bundles_at_their_hash_are_pruned_after_the_evidence_gate(self) -> None: archive = self.home / "archive/glaeda-disk" archive.mkdir(parents=True) From f43fca4cb64b40a79a7e71f4661c5159b69aa1f2 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 26 Oct 2026 17:46:23 -0700 Subject: [PATCH 25/28] glaeda-disk: a bundled mirror's git metadata is checked again after the rename; references rechecked Addresses the eighth Codex review of #1625: 1. mirror_rename_recheck runs as remove()'s after_rename: once the tree is renamed aside, a hash of every git dir's HEAD, packed-refs, refs, logs and worktrees (mirror_git_digest, read relative to the item) must equal the one taken before bundling, or the tree is renamed back and kept (changed:in-use). mirror_moved compares the same digest too. 2. mirror_moved re-runs referenced_main_checkout for git-disposable families. 3. files_strict also refuses a symlinked or special file, which is_file() readers would skip. Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- scripts/glaeda-disk | 59 +++++++++++++++++++++++++++++++++++-- scripts/test-glaeda-disk.py | 41 +++++++++++++++++++++++++- 2 files changed, 96 insertions(+), 4 deletions(-) diff --git a/scripts/glaeda-disk b/scripts/glaeda-disk index 70da59451..28a1783bf 100755 --- a/scripts/glaeda-disk +++ b/scripts/glaeda-disk @@ -4316,6 +4316,12 @@ def files_strict(d: Path) -> list[Path] | None: for dp, dirs, fs in os.walk(d, onerror=errors.append): if any(os.path.islink(os.path.join(dp, x)) for x in dirs): # os.walk would not enter it: unread return None + for f in fs: # a symlink or special file is skipped by is_file() readers: unread too + try: + if not stat.S_ISREG(os.lstat(os.path.join(dp, f)).st_mode): + return None + except OSError: + return None found += [Path(dp) / f for f in fs] return None if errors or d.is_symlink() else found @@ -4673,6 +4679,47 @@ def _work_tree_dirty(root: Path) -> str: return "" +def mirror_git_digest(root: Path, gitdirs: list[Path], at: Path | None = None) -> str | None: + """A hash of every git dir's HEAD, packed-refs, refs/, logs/ and worktrees/ (each per-worktree HEAD and + reflog), read as files relative to root, so it can be read again at `at` once root was renamed there.""" + h = hashlib.sha256() + for g in sorted(gitdirs): + base = (at or root) / os.path.relpath(g, root) + for name in ("HEAD", "packed-refs"): + try: + h.update(f"{g}/{name}\0".encode() + (base / name).read_bytes()) + except FileNotFoundError: + h.update(f"{g}/{name}\0-".encode()) + except OSError: + return None + for sub_dir in ("refs", "logs", "worktrees"): + if (files := files_strict(base / sub_dir)) is None: + return None + for f in sorted(files): + try: + h.update(f"{f.relative_to(base)}\0".encode() + f.read_bytes()) + except OSError: + return None + return h.hexdigest() + + +def mirror_rename_recheck(path: Path, recheck: tuple): + """after_rename for a bundled mirror: once the tree is out of the way, its git metadata must still be what + archive_mirror bundled; otherwise it is put back and kept (SlotTaken: changed:in-use).""" + doomed = path.with_name(f"{DELETING_PREFIX}{path.name}-{os.getpid()}") + gitdirs, digest = recheck[0], recheck[4] + + def check() -> None: + if digest is not None and mirror_git_digest(path, gitdirs, at=doomed) == digest: + return + try: + rename_noreplace(doomed, path) + except OSError as e: + raise SlotTaken(f"{path.name}: git changed after bundling; left aside as {doomed.name} ({e.strerror})") + raise SlotTaken(f"{path.name}: git changed after bundling; put back") + return check + + def mirror_git_snapshot(gitdirs: list[Path], trees: list[Path]) -> list[str | None]: """Every ref, HEAD and reflog size of the item's repositories and the status of its work trees: compared before and after archive_mirror's bundles, so a commit made meanwhile keeps the tree.""" @@ -4798,24 +4845,25 @@ def archive_mirror(path: Path) -> tuple[str, list[Path], tuple]: """Before a mirror item goes: bundle_unpushed for every repository mirror_repos finds. A tree with uncommitted work, or whose repositories cannot all be found or bundled, is kept. ("" or why it must stay, the bundles written, what mirror_moved rechecks).""" - none: tuple = ([], [], [], {}) + none: tuple = ([], [], [], {}, None) why, gitdirs, trees = mirror_repos(path) if why: return why, [], none snap = mirror_git_snapshot(list(gitdirs.values()), trees) + digest = mirror_git_digest(path, list(gitdirs.values())) bundles: list[Path] = [] relied: dict[Path, list[str]] = {} for key, g in sorted(gitdirs.items()): if (why := bundle_unpushed(g, key, bundles, relied.setdefault(g, []))): return why, bundles, none - return "", bundles, (list(gitdirs.values()), trees, snap, relied) + return "", bundles, (list(gitdirs.values()), trees, snap, relied, digest) def mirror_moved(fam: Family, path: Path, recheck: tuple) -> str: """Right before a bundled mirror goes, everything the bundling took time over is read again: process evidence, the family's owner and artifact references, and its repositories, found afresh (the same set, every work tree clean) and as at bundle time, and the servers, asked again for what the bundles left to them.""" - gitdirs, trees, snap, relied = recheck + gitdirs, trees, snap, relied, digest = recheck for g, commits in relied.items(): # the slow network first, then git, then the quick process checks last if (why := server_still_holds(g, commits)): return why @@ -4825,6 +4873,10 @@ def mirror_moved(fam: Family, path: Path, recheck: tuple) -> str: return f"its repositories changed while they were bundled{': ' + why if why else ''}" if None in snap or mirror_git_snapshot(gitdirs, trees) != snap: return "its git refs or work trees changed while they were bundled" + if digest is None or mirror_git_digest(path, gitdirs) != digest: + return "its git metadata changed while it was bundled" + if fam.git_disposable and (ref := referenced_main_checkout(path, nested=fam.nested_git)): + return f"referenced while its git work was bundled: {ref}" if fam.retired_only and not owner_retired(fam): return "its owner loaded while its git work was bundled" if fam.reference_root is not None and artifact_referenced(path, fresh_artifact_references(fam.reference_root)): @@ -6849,6 +6901,7 @@ def apply(items: list[Item], families: dict[str, Family], receipt: Path, if fresh_layout == "rustup-toolchain" else None) remove(p, after_rename=cmux_next_ci_slot_recheck(fam, p, floor=cmux_next_ci_floor_slot(fam, p)) if whole_state + else mirror_rename_recheck(p, bundled[2]) if mirror_due(fam, p, idle_now) else None if stale_hash is None else lambda h=stale_hash: h.unlink(missing_ok=True)) gone = True if fam.id == CMUX_NEXT_CI_FAMILY and p.parent.parent.parent in tuple( diff --git a/scripts/test-glaeda-disk.py b/scripts/test-glaeda-disk.py index e791c2fa7..5c3bb5a4e 100644 --- a/scripts/test-glaeda-disk.py +++ b/scripts/test-glaeda-disk.py @@ -1518,7 +1518,7 @@ def git_state(f, path): # a cargo target dir too large to search mock.patch.object(gd, "process_evidence", return_value=late), \ mock.patch.object(gd, "command_lines", return_value=late[1]), \ mock.patch.object(gd, "HOME", self.root / "home"), \ - mock.patch.object(gd, "archive_mirror", return_value=("", [], ([], [], [], {}))), \ + mock.patch.object(gd, "archive_mirror", return_value=("", [], ([], [], [], {}, gd.mirror_git_digest(Path("/"), [])))), \ mock.patch.object(gd, "mirror_moved", return_value=""): # bundling: the cmux-job tests gd.apply(reclaim, {fam.id: fam}, receipt, None, 24) self.assertEqual(sorted(p.name for p in cache.iterdir()), @@ -6091,6 +6091,45 @@ def test_an_unreadable_outside_linked_worktree_keeps_the_mirror(self) -> None: self.assertIn("unreadable linked worktree", why) self.assertEqual(bundles, []) + def test_a_commit_after_the_last_recheck_puts_the_mirror_back(self) -> None: + unit, clone, env = self.mirror_unit() + real = gd.mirror_moved + + def late(fam, path, recheck): + out = real(fam, path, recheck) + self.git(clone, "commit", "-q", "--allow-empty", "-m", "made after the recheck", env=env) + return out + with mock.patch.object(gd, "mirror_moved", late): + rec, _ = self.apply_mirror(unit) + self.assertEqual(rec["outcome"], "changed:in-use") + self.assertTrue((clone / ".git").is_dir(), "renamed back to its path") + self.assertEqual([p.name for p in unit.parent.iterdir() if p.name.startswith(gd.DELETING_PREFIX)], []) + + def test_a_checkout_referenced_while_bundling_keeps_the_mirror(self) -> None: + unit, clone, _ = self.mirror_unit() + bundled = [] + real = gd.archive_mirror + + def archive(path): + bundled.append(path) # a tool names the checkout while its git work is bundled + return real(path) + with mock.patch.object(gd, "referenced_main_checkout", + lambda path, nested=False: "named by ~/.local/bin/tool" if bundled else ""), \ + mock.patch.object(gd, "archive_mirror", archive): + rec, _ = self.apply_mirror(unit) + self.assertEqual(rec["outcome"], "changed:in-use") + self.assertTrue(any("referenced while its git work was bundled" in r for r in rec["reasons"]), rec["reasons"]) + self.assertTrue((clone / ".git").is_dir()) + + def test_a_symlinked_or_special_reflog_file_keeps_the_mirror(self) -> None: + unit, clone, _ = self.mirror_unit() + log = clone / ".git/logs/HEAD" + log.rename(self.home / "head-log") + log.symlink_to(self.home / "head-log") + self.assertIsNone(gd.files_strict(clone / ".git/logs")) + self.assertIsNone(gd.local_tips(clone / ".git")) + self.assertTrue(gd.archive_mirror(unit)[0]) + def test_only_manifest_listed_bundles_at_their_hash_are_pruned_after_the_evidence_gate(self) -> None: archive = self.home / "archive/glaeda-disk" archive.mkdir(parents=True) From 204c98be338b42a0c352c1f8a6f9cc4ad269ccc3 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 26 Oct 2026 18:05:20 -0700 Subject: [PATCH 26/28] glaeda-disk: after the rename, a bundled mirror gets the whole pre-bundle check again; kept trees are marked Addresses the ninth Codex review of #1625 (replaces the metadata digest): 1, 2. mirror_rename_recheck runs mirror_repos on the renamed tree (a fresh strict scan of repositories and linked worktrees, every work tree clean, staged and untracked included, and unlocked) and compares each repository's local tips with what was bundled or left to the servers; any new repository or commit, or any error, puts the tree back. 3. local_tips refuses a symlinked or special HEAD or packed-refs. 4. A tree that cannot be renamed back gets .glaeda-disk-keep, which finish_deletion and the interrupted-deletions family (keep_verdict) honour, so nothing deletes it. Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- scripts/glaeda-disk | 95 ++++++++++++++++++++++--------------- scripts/test-glaeda-disk.py | 64 ++++++++++++++++++++++++- 2 files changed, 120 insertions(+), 39 deletions(-) diff --git a/scripts/glaeda-disk b/scripts/glaeda-disk index 28a1783bf..18bfc6253 100755 --- a/scripts/glaeda-disk +++ b/scripts/glaeda-disk @@ -3613,6 +3613,8 @@ def finish_deletion(path: Path) -> None: raise OSError("refusing a mount point") if not DELETING_RE.fullmatch(path.name): raise OSError("refusing a tree without the deleting name") + if os.path.lexists(path / MIRROR_KEEP): + raise OSError("refusing a tree glaeda-disk kept: its git work changed after bundling") if path.is_dir(): shutil.rmtree(path, onerror=_chmod_retry) else: @@ -3939,6 +3941,8 @@ def keep_verdict(fam: "Family", path: Path) -> str: if fam.id == "cmux-seeds": return seed_verdict(path) if fam.id == INTERRUPTED: + if os.path.lexists(path / MIRROR_KEEP): + return "glaeda-disk kept it: its git work changed after bundling" return deletion_owner_live(path) if fam.discover and stat_layout(os.fspath(path)) == "rustup-toolchain": return rustup_verdict(path) @@ -4335,6 +4339,15 @@ def local_tips(gitdir: Path, worktree_only: bool = False) -> list[str] | None: ones, and every reflog entry (a commit left behind by `submodule update` lives only there). With worktree_only, only what a linked worktree's git dir holds for itself: HEAD, its per-worktree refs and its reflogs. None when they cannot all be read.""" + for name in ("HEAD", "packed-refs"): # git reads these through a symlink; a hidden one is unreadable + try: + if not stat.S_ISREG(os.lstat(gitdir / name).st_mode): + return None + except FileNotFoundError: + if name == "HEAD": + return None + except OSError: + return None refs = _git_dir_run(gitdir, "for-each-ref", "--format=%(refname)%09%(objectname)%09%(*objectname)", *(WORKTREE_REFS if worktree_only else ())) @@ -4590,11 +4603,8 @@ def bundle_unpushed(gitdir: Path, key: str, written: list[Path], relied: list[st and what `fetch --negotiate-only` acknowledges; local remote-tracking refs are only candidates to ask about), and every branch or tag the servers do not have at that commit. The commits left to the servers go to relied, for server_still_holds. "" or why it failed.""" - if _git_dir_run(gitdir, "for-each-ref", "--count=1") == "" and _git_dir_run( - gitdir, "rev-parse", "-q", "--verify", "HEAD") is None and _git_dir_run( - gitdir, "symbolic-ref", "-q", "HEAD") is not None and files_strict(gitdir / "logs") == [] \ - and not os.path.lexists(gitdir / "worktrees"): - return "" # unborn: no ref, reflog or linked worktree, HEAD a branch with no commit yet; nothing to keep + if repo_unborn(gitdir): + return "" # nothing to keep tips = local_tips(gitdir) for wt in [p for p in (gitdir / "worktrees").glob("*") if p.is_dir()] if tips is not None else []: own = local_tips(wt, worktree_only=True) @@ -4679,44 +4689,53 @@ def _work_tree_dirty(root: Path) -> str: return "" -def mirror_git_digest(root: Path, gitdirs: list[Path], at: Path | None = None) -> str | None: - """A hash of every git dir's HEAD, packed-refs, refs/, logs/ and worktrees/ (each per-worktree HEAD and - reflog), read as files relative to root, so it can be read again at `at` once root was renamed there.""" - h = hashlib.sha256() - for g in sorted(gitdirs): - base = (at or root) / os.path.relpath(g, root) - for name in ("HEAD", "packed-refs"): - try: - h.update(f"{g}/{name}\0".encode() + (base / name).read_bytes()) - except FileNotFoundError: - h.update(f"{g}/{name}\0-".encode()) - except OSError: - return None - for sub_dir in ("refs", "logs", "worktrees"): - if (files := files_strict(base / sub_dir)) is None: - return None - for f in sorted(files): - try: - h.update(f"{f.relative_to(base)}\0".encode() + f.read_bytes()) - except OSError: - return None - return h.hexdigest() +MIRROR_KEEP = ".glaeda-disk-keep" # in an aside tree: finish_deletion and interrupted-deletions never delete it + + +def repo_unborn(gitdir: Path) -> bool: + """No ref, reflog or linked worktree, and HEAD a branch with no commit yet: a repository with nothing in it.""" + return (_git_dir_run(gitdir, "for-each-ref", "--count=1") == "" + and _git_dir_run(gitdir, "rev-parse", "-q", "--verify", "HEAD") is None + and _git_dir_run(gitdir, "symbolic-ref", "-q", "HEAD") is not None + and files_strict(gitdir / "logs") == [] and not os.path.lexists(gitdir / "worktrees")) + + +def mirror_tips(root: Path, gitdirs: dict[str, Path]) -> dict[str, list[str]] | None: + """Every commit each repository points at on its own (local_tips: refs, HEAD, reflogs, its worktrees'), + keyed by the git dir's path relative to root, so the same item can be read again after a rename.""" + tips: dict[str, list[str]] = {} + for g in gitdirs.values(): + t = [] if repo_unborn(g) else local_tips(g) + for wt in [p for p in (g / "worktrees").glob("*") if p.is_dir()] if t is not None else []: + own = local_tips(wt, worktree_only=True) + t = None if own is None else sorted({*t, *own}) + if t is None: + return None + tips[os.path.relpath(g, root)] = t + return tips def mirror_rename_recheck(path: Path, recheck: tuple): - """after_rename for a bundled mirror: once the tree is out of the way, its git metadata must still be what - archive_mirror bundled; otherwise it is put back and kept (SlotTaken: changed:in-use).""" + """after_rename for a bundled mirror: the renamed tree gets the whole check archive_mirror made before its + bundles again (a fresh strict scan of repositories and work trees, each clean and unlocked), and must hold + the same repositories with no commit beyond what was bundled or left to the servers. Otherwise it goes back + to its path, or, when that fails, stays aside with MIRROR_KEEP so nothing deletes it.""" doomed = path.with_name(f"{DELETING_PREFIX}{path.name}-{os.getpid()}") - gitdirs, digest = recheck[0], recheck[4] + before = recheck[4] def check() -> None: - if digest is not None and mirror_git_digest(path, gitdirs, at=doomed) == digest: + why, gitdirs, _ = mirror_repos(doomed) + now = None if why else mirror_tips(doomed, gitdirs) + if now is not None and before is not None and set(now) == set(before) \ + and all(set(t) <= set(before[k]) for k, t in now.items()): return try: rename_noreplace(doomed, path) except OSError as e: - raise SlotTaken(f"{path.name}: git changed after bundling; left aside as {doomed.name} ({e.strerror})") - raise SlotTaken(f"{path.name}: git changed after bundling; put back") + with contextlib.suppress(OSError): + (doomed / MIRROR_KEEP).write_text(f"kept by glaeda-disk {os.getpid()}: {why or 'git changed'}\n") + raise SlotTaken(f"{path.name}: changed after bundling; kept aside as {doomed.name} ({e.strerror})") + raise SlotTaken(f"{path.name}: changed after bundling ({why or 'git changed'}); put back") return check @@ -4850,20 +4869,22 @@ def archive_mirror(path: Path) -> tuple[str, list[Path], tuple]: if why: return why, [], none snap = mirror_git_snapshot(list(gitdirs.values()), trees) - digest = mirror_git_digest(path, list(gitdirs.values())) + tips = mirror_tips(path, gitdirs) # what the bundles and the servers cover, for mirror_rename_recheck + if tips is None: + return "unreadable refs", [], none bundles: list[Path] = [] relied: dict[Path, list[str]] = {} for key, g in sorted(gitdirs.items()): if (why := bundle_unpushed(g, key, bundles, relied.setdefault(g, []))): return why, bundles, none - return "", bundles, (list(gitdirs.values()), trees, snap, relied, digest) + return "", bundles, (list(gitdirs.values()), trees, snap, relied, tips) def mirror_moved(fam: Family, path: Path, recheck: tuple) -> str: """Right before a bundled mirror goes, everything the bundling took time over is read again: process evidence, the family's owner and artifact references, and its repositories, found afresh (the same set, every work tree clean) and as at bundle time, and the servers, asked again for what the bundles left to them.""" - gitdirs, trees, snap, relied, digest = recheck + gitdirs, trees, snap, relied, _ = recheck for g, commits in relied.items(): # the slow network first, then git, then the quick process checks last if (why := server_still_holds(g, commits)): return why @@ -4873,8 +4894,6 @@ def mirror_moved(fam: Family, path: Path, recheck: tuple) -> str: return f"its repositories changed while they were bundled{': ' + why if why else ''}" if None in snap or mirror_git_snapshot(gitdirs, trees) != snap: return "its git refs or work trees changed while they were bundled" - if digest is None or mirror_git_digest(path, gitdirs) != digest: - return "its git metadata changed while it was bundled" if fam.git_disposable and (ref := referenced_main_checkout(path, nested=fam.nested_git)): return f"referenced while its git work was bundled: {ref}" if fam.retired_only and not owner_retired(fam): diff --git a/scripts/test-glaeda-disk.py b/scripts/test-glaeda-disk.py index 5c3bb5a4e..62acdaf10 100644 --- a/scripts/test-glaeda-disk.py +++ b/scripts/test-glaeda-disk.py @@ -1518,7 +1518,8 @@ def git_state(f, path): # a cargo target dir too large to search mock.patch.object(gd, "process_evidence", return_value=late), \ mock.patch.object(gd, "command_lines", return_value=late[1]), \ mock.patch.object(gd, "HOME", self.root / "home"), \ - mock.patch.object(gd, "archive_mirror", return_value=("", [], ([], [], [], {}, gd.mirror_git_digest(Path("/"), [])))), \ + mock.patch.object(gd, "archive_mirror", return_value=("", [], ([], [], [], {}, {}))), \ + mock.patch.object(gd, "mirror_rename_recheck", lambda p, r: None), \ mock.patch.object(gd, "mirror_moved", return_value=""): # bundling: the cmux-job tests gd.apply(reclaim, {fam.id: fam}, receipt, None, 24) self.assertEqual(sorted(p.name for p in cache.iterdir()), @@ -6130,6 +6131,67 @@ def test_a_symlinked_or_special_reflog_file_keeps_the_mirror(self) -> None: self.assertIsNone(gd.local_tips(clone / ".git")) self.assertTrue(gd.archive_mirror(unit)[0]) + def recheck_after_rename(self, change, unit: Path | None = None) -> tuple[dict, Path]: + """apply with `change(doomed)` run on the renamed-aside tree, right before the post-rename recheck.""" + unit = unit or self.mirror_unit()[0] + real = gd.mirror_rename_recheck + + def wrapped(path, recheck): + inner = real(path, recheck) + + def run(): + change(path.with_name(f"{gd.DELETING_PREFIX}{path.name}-{os.getpid()}")) + inner() + return run + with mock.patch.object(gd, "mirror_rename_recheck", wrapped): + rec, _ = self.apply_mirror(unit) + return rec, unit + + def test_work_staged_after_the_last_check_puts_the_mirror_back(self) -> None: + def stage(doomed): + (doomed / "slot-1/new.txt").write_text("x") + subprocess.run(["git", "-C", str(doomed / "slot-1"), "add", "new.txt"], check=True) + rec, unit = self.recheck_after_rename(stage) + self.assertEqual(rec["outcome"], "changed:in-use") + self.assertTrue((unit / "slot-1/new.txt").exists()) + + def test_a_repository_created_after_the_last_check_puts_the_mirror_back(self) -> None: + def create(doomed): + subprocess.run(["git", "init", "-q", str(doomed / "slot-9")], check=True) + rec, unit = self.recheck_after_rename(create) + self.assertEqual(rec["outcome"], "changed:in-use") + self.assertTrue((unit / "slot-9/.git").is_dir()) + + def test_a_symlinked_packed_refs_is_unreadable(self) -> None: + _, clone, _ = self.mirror_unit() + (clone / ".git/packed-refs").write_text("") + (clone / ".git/packed-refs").rename(self.home / "packed") + (clone / ".git/packed-refs").symlink_to(self.home / "packed") + self.assertIsNone(gd.local_tips(clone / ".git")) + + def test_a_tree_that_cannot_go_back_is_kept_aside_and_never_finished(self) -> None: + unit, clone, env = self.mirror_unit() + real_rename = gd.rename_noreplace + + def blocked(src, dst): + raise OSError(errno.EEXIST, "File exists") + + def commit(doomed): + subprocess.run(["git", "-C", str(doomed / "slot-1"), "commit", "-q", "--allow-empty", "-m", "late"], + check=True, env=env) + with mock.patch.object(gd, "rename_noreplace", blocked): + rec, _ = self.recheck_after_rename(commit, unit) + self.assertEqual(rec["outcome"], "changed:in-use") + aside = [p for p in unit.parent.iterdir() if p.name.startswith(gd.DELETING_PREFIX)] + self.assertEqual(len(aside), 1) + self.assertTrue((aside[0] / gd.MIRROR_KEEP).exists()) + with self.assertRaises(OSError): + gd.finish_deletion(aside[0]) + fam = gd.Family(gd.INTERRUPTED, unit.parent, True, "x") + self.assertIn("kept it", gd.keep_verdict(fam, aside[0])) + self.assertTrue((aside[0] / "slot-1/.git").is_dir()) + del real_rename + def test_only_manifest_listed_bundles_at_their_hash_are_pruned_after_the_evidence_gate(self) -> None: archive = self.home / "archive/glaeda-disk" archive.mkdir(parents=True) From 2276d1882db4ae156417c3d40fc362c596111b51 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 26 Oct 2026 18:18:55 -0700 Subject: [PATCH 27/28] glaeda-disk: unborn means strictly empty; post-rename recheck compares refs; keep marker first Addresses the tenth Codex review of #1625: 1. repo_unborn needs a regular HEAD, no packed-refs and an empty refs/ read strictly. 2. mirror_tips records each repository's local refs by name as well as its tips; the post-rename recheck requires the refs to be unchanged, so a branch or tag named after the last check puts the tree back. 3. The keep marker is a sidecar written before the rename (a failed write leaves the tree in place) and removed only once the recheck passes, so a kept aside tree always carries it. Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- scripts/glaeda-disk | 49 +++++++++++++++++++++++++------------ scripts/test-glaeda-disk.py | 26 +++++++++++++++++++- 2 files changed, 59 insertions(+), 16 deletions(-) diff --git a/scripts/glaeda-disk b/scripts/glaeda-disk index 18bfc6253..fc11fe3b0 100755 --- a/scripts/glaeda-disk +++ b/scripts/glaeda-disk @@ -3613,7 +3613,7 @@ def finish_deletion(path: Path) -> None: raise OSError("refusing a mount point") if not DELETING_RE.fullmatch(path.name): raise OSError("refusing a tree without the deleting name") - if os.path.lexists(path / MIRROR_KEEP): + if os.path.lexists(mirror_keep_marker(path)): raise OSError("refusing a tree glaeda-disk kept: its git work changed after bundling") if path.is_dir(): shutil.rmtree(path, onerror=_chmod_retry) @@ -3941,7 +3941,7 @@ def keep_verdict(fam: "Family", path: Path) -> str: if fam.id == "cmux-seeds": return seed_verdict(path) if fam.id == INTERRUPTED: - if os.path.lexists(path / MIRROR_KEEP): + if os.path.lexists(mirror_keep_marker(path)): return "glaeda-disk kept it: its git work changed after bundling" return deletion_owner_live(path) if fam.discover and stat_layout(os.fspath(path)) == "rustup-toolchain": @@ -4689,29 +4689,41 @@ def _work_tree_dirty(root: Path) -> str: return "" -MIRROR_KEEP = ".glaeda-disk-keep" # in an aside tree: finish_deletion and interrupted-deletions never delete it +MIRROR_KEEP = ".glaeda-disk-keep-" # next to an aside tree: finish_deletion and interrupted-deletions never delete it + + +def mirror_keep_marker(aside: Path) -> Path: + return aside.with_name(MIRROR_KEEP + aside.name) def repo_unborn(gitdir: Path) -> bool: """No ref, reflog or linked worktree, and HEAD a branch with no commit yet: a repository with nothing in it.""" - return (_git_dir_run(gitdir, "for-each-ref", "--count=1") == "" + try: + if not stat.S_ISREG(os.lstat(gitdir / "HEAD").st_mode) or os.path.lexists(gitdir / "packed-refs"): + return False # a symlinked HEAD or any packed-refs at all: not judged empty + except OSError: + return False + return (files_strict(gitdir / "refs") == [] and _git_dir_run(gitdir, "for-each-ref", "--count=1") == "" and _git_dir_run(gitdir, "rev-parse", "-q", "--verify", "HEAD") is None and _git_dir_run(gitdir, "symbolic-ref", "-q", "HEAD") is not None and files_strict(gitdir / "logs") == [] and not os.path.lexists(gitdir / "worktrees")) -def mirror_tips(root: Path, gitdirs: dict[str, Path]) -> dict[str, list[str]] | None: - """Every commit each repository points at on its own (local_tips: refs, HEAD, reflogs, its worktrees'), - keyed by the git dir's path relative to root, so the same item can be read again after a rename.""" - tips: dict[str, list[str]] = {} +def mirror_tips(root: Path, gitdirs: dict[str, Path]) -> dict[str, tuple[list[str], str]] | None: + """Every commit each repository points at on its own (local_tips: refs, HEAD, reflogs, its worktrees'), and + its local refs by name, keyed by the git dir's path relative to root, so the same item can be read again + after a rename.""" + tips: dict[str, tuple[list[str], str]] = {} for g in gitdirs.values(): t = [] if repo_unborn(g) else local_tips(g) for wt in [p for p in (g / "worktrees").glob("*") if p.is_dir()] if t is not None else []: own = local_tips(wt, worktree_only=True) t = None if own is None else sorted({*t, *own}) - if t is None: + refs = _git_dir_run(g, "for-each-ref", "--format=%(refname) %(objectname)") + if t is None or refs is None: return None - tips[os.path.relpath(g, root)] = t + tips[os.path.relpath(g, root)] = (t, "\n".join( + ln for ln in refs.splitlines() if not ln.startswith(("refs/remotes/", ARCHIVE_REF)))) return tips @@ -4719,22 +4731,29 @@ def mirror_rename_recheck(path: Path, recheck: tuple): """after_rename for a bundled mirror: the renamed tree gets the whole check archive_mirror made before its bundles again (a fresh strict scan of repositories and work trees, each clean and unlocked), and must hold the same repositories with no commit beyond what was bundled or left to the servers. Otherwise it goes back - to its path, or, when that fails, stays aside with MIRROR_KEEP so nothing deletes it.""" + to its path, or, when that fails, stays aside: its keep marker is written before the rename (if that write + fails, nothing is renamed or deleted) and removed only once the check passes.""" doomed = path.with_name(f"{DELETING_PREFIX}{path.name}-{os.getpid()}") before = recheck[4] + marker = mirror_keep_marker(doomed) + marker.write_text(f"kept by glaeda-disk {os.getpid()} until its git work is checked again\n") # raises: kept def check() -> None: why, gitdirs, _ = mirror_repos(doomed) now = None if why else mirror_tips(doomed, gitdirs) if now is not None and before is not None and set(now) == set(before) \ - and all(set(t) <= set(before[k]) for k, t in now.items()): - return + and all(set(t) <= set(before[k][0]) and refs == before[k][1] for k, (t, refs) in now.items()): + try: + marker.unlink() + return + except OSError as e: + why = f"its keep marker could not be removed ({e.strerror})" try: rename_noreplace(doomed, path) except OSError as e: - with contextlib.suppress(OSError): - (doomed / MIRROR_KEEP).write_text(f"kept by glaeda-disk {os.getpid()}: {why or 'git changed'}\n") raise SlotTaken(f"{path.name}: changed after bundling; kept aside as {doomed.name} ({e.strerror})") + with contextlib.suppress(OSError): + marker.unlink() raise SlotTaken(f"{path.name}: changed after bundling ({why or 'git changed'}); put back") return check diff --git a/scripts/test-glaeda-disk.py b/scripts/test-glaeda-disk.py index 62acdaf10..5d780514e 100644 --- a/scripts/test-glaeda-disk.py +++ b/scripts/test-glaeda-disk.py @@ -6184,7 +6184,7 @@ def commit(doomed): self.assertEqual(rec["outcome"], "changed:in-use") aside = [p for p in unit.parent.iterdir() if p.name.startswith(gd.DELETING_PREFIX)] self.assertEqual(len(aside), 1) - self.assertTrue((aside[0] / gd.MIRROR_KEEP).exists()) + self.assertTrue(gd.mirror_keep_marker(aside[0]).exists()) with self.assertRaises(OSError): gd.finish_deletion(aside[0]) fam = gd.Family(gd.INTERRUPTED, unit.parent, True, "x") @@ -6192,6 +6192,30 @@ def commit(doomed): self.assertTrue((aside[0] / "slot-1/.git").is_dir()) del real_rename + def test_a_symlinked_packed_refs_is_never_judged_empty(self) -> None: + bare = self.home / "bare.git" + subprocess.run(["git", "init", "-q", "--bare", str(bare)], check=True) + (self.home / "packed").write_text("# pack-refs with: peeled\n") + (bare / "packed-refs").symlink_to(self.home / "packed") + self.assertFalse(gd.repo_unborn(bare)) + self.assertTrue(gd.bundle_unpushed(bare, "k", [])) + + def test_a_branch_named_after_the_last_check_puts_the_mirror_back(self) -> None: + def name(doomed): + subprocess.run(["git", "-C", str(doomed / "slot-1"), "tag", "made-late", "HEAD"], check=True) + rec, unit = self.recheck_after_rename(name) + self.assertEqual(rec["outcome"], "changed:in-use") + self.assertIn("made-late", subprocess.run(["git", "-C", str(unit / "slot-1"), "tag"], capture_output=True, + text=True, check=True).stdout) + + def test_a_keep_marker_that_cannot_be_written_keeps_the_tree_in_place(self) -> None: + unit, clone, _ = self.mirror_unit() + with mock.patch.object(gd, "mirror_keep_marker", lambda aside: self.home / "no-such-dir" / "marker"): + rec, _ = self.apply_mirror(unit) + self.assertTrue(rec["outcome"].startswith("incomplete:"), rec["outcome"]) + self.assertTrue((clone / ".git").is_dir()) + self.assertEqual([p.name for p in unit.parent.iterdir() if p.name.startswith(gd.DELETING_PREFIX)], []) + def test_only_manifest_listed_bundles_at_their_hash_are_pruned_after_the_evidence_gate(self) -> None: archive = self.home / "archive/glaeda-disk" archive.mkdir(parents=True) From 4b654857c3ccb790b1369b38910a3ef079766f58 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Mon, 26 Oct 2026 18:26:12 -0700 Subject: [PATCH 28/28] glaeda-disk: a mirror whose repositories have linked worktrees is kept Addresses the eleventh Codex review of #1625: worktree records name absolute paths, so once the item is renamed aside its linked worktrees cannot be found and checked again. Any repository in a mirror item with linked worktree records (inside or outside the item) now keeps the item; the post-rename recheck finds the same records and keeps it too. Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-85ca59f74f02d81734c3 --- scripts/glaeda-disk | 21 ++++----------------- scripts/test-glaeda-disk.py | 20 ++++++++++---------- 2 files changed, 14 insertions(+), 27 deletions(-) diff --git a/scripts/glaeda-disk b/scripts/glaeda-disk index fc11fe3b0..77feef953 100755 --- a/scripts/glaeda-disk +++ b/scripts/glaeda-disk @@ -4793,7 +4793,7 @@ def _linked_trees_strict(gitdir: Path) -> list[Path] | None: return None -def mirror_repos(path: Path, linked_inside: tuple[Path, ...] = ()) -> tuple[str, dict[str, Path], list[Path]]: +def mirror_repos(path: Path) -> tuple[str, dict[str, Path], list[Path]]: """Every repository a mirror item takes along, found as _nested_loss finds them (checkouts, bare repositories, a linked worktree's repository inside it, checkouts in ignored directories, submodule stores and initialized submodules, recursively), and its work trees, each checked clean right now. @@ -4801,7 +4801,7 @@ def mirror_repos(path: Path, linked_inside: tuple[Path, ...] = ()) -> tuple[str, queue = checkouts(path, NESTED_DEPTH, NESTED_BUDGET, strict=True) # an unreadable directory may hide one if queue is None: return "too deep, large or unreadable to find every checkout", {}, [] - queue, inside = [*queue, *linked_inside], os.path.realpath(path) + "/" + queue, inside = list(queue), os.path.realpath(path) + "/" gitdirs: dict[str, Path] = {} trees: list[Path] = [] seen: set[str] = set() @@ -4858,24 +4858,11 @@ def mirror_repos(path: Path, linked_inside: tuple[Path, ...] = ()) -> tuple[str, dirnames[:] = ["modules"] if "modules" in dirnames else [] if [u for u in unreadable if os.path.lexists(u)]: # a missing modules/ is no submodule store return f"unreadable submodule store {unreadable[0]}", {}, [] - found: list[Path] = [] for key, g in gitdirs.items(): if (linked := _linked_trees_strict(g)) is None: return f"{key}: unreadable linked worktree records", {}, [] - present: list[Path] = [] - for t in linked: - try: # os.stat, not exists(): a tree we cannot read (EACCES, EIO) is not an absent one - os.stat(t) - present.append(t) - except FileNotFoundError: - continue - except OSError: - return f"{key}: unreadable linked worktree {t}", {}, [] - if any(not (os.path.realpath(t) + "/").startswith(inside) for t in present): - return f"{key}: a worktree outside this item uses it", {}, [] - found += [t for t in present if os.path.realpath(t) not in seen] - if found: # a linked worktree inside the item, even in ignored output: checked clean and unlocked too - return mirror_repos(path, (*linked_inside, *found)) + if linked: # the records name absolute paths, so after the rename its worktrees can't be checked again + return f"{key}: has linked worktrees ({', '.join(map(os.fspath, linked[:3]))}); kept", {}, [] return "", gitdirs, sorted(trees) diff --git a/scripts/test-glaeda-disk.py b/scripts/test-glaeda-disk.py index 5d780514e..453013b03 100644 --- a/scripts/test-glaeda-disk.py +++ b/scripts/test-glaeda-disk.py @@ -5963,7 +5963,8 @@ def test_a_locked_worktree_of_a_repository_inside_the_mirror_keeps_it(self) -> N self.git(clone, "worktree", "add", "-q", "--detach", "--lock", str(unit / "slot-2"), env=env) rec, _ = self.apply_mirror(unit) self.assertEqual(rec["outcome"], "blocked:bundle-failed") - self.assertTrue(any("locked worktree" in r for r in rec["reasons"]), rec["reasons"]) + self.assertTrue(any("locked worktree" in r or "has linked worktrees" in r for r in rec["reasons"]), + rec["reasons"]) self.assertTrue((unit / "slot-2/.git").is_file()) def test_a_commit_made_while_the_servers_are_asked_keeps_the_mirror(self) -> None: @@ -6054,21 +6055,20 @@ def test_an_unreadable_reflog_directory_keeps_the_mirror(self) -> None: self.assertTrue(why) self.assertEqual(bundles, []) - def test_a_dirty_or_locked_worktree_in_ignored_output_keeps_the_mirror(self) -> None: + def test_a_linked_worktree_inside_the_mirror_keeps_it_however_clean(self) -> None: + """Its record names an absolute path, so once the item is renamed the worktree cannot be checked again.""" unit, clone, env = self.mirror_unit() with (clone / ".git/info/exclude").open("a") as f: f.write("target/\n") wt = clone / "target/wt" self.git(clone, "worktree", "add", "-q", "--detach", str(wt), env=env) (wt / "edit.txt").write_text("uncommitted") - why, _, _ = gd.archive_mirror(unit) - self.assertIn("uncommitted", why) + self.assertTrue(gd.archive_mirror(unit)[0]) (wt / "edit.txt").unlink() - self.git(clone, "worktree", "lock", str(wt), env=env) - why, _, _ = gd.archive_mirror(unit) - self.assertIn("locked worktree", why) - self.git(clone, "worktree", "unlock", str(wt), env=env) - self.assertEqual(gd.archive_mirror(unit)[0], "") + self.assertIn("has linked worktrees", gd.archive_mirror(unit)[0]) + rec, _ = self.apply_mirror(unit) + self.assertEqual(rec["outcome"], "blocked:bundle-failed") + self.assertTrue((wt / ".git").is_file()) def test_a_symlinked_reflog_directory_keeps_the_mirror(self) -> None: unit, clone, _ = self.mirror_unit() @@ -6089,7 +6089,7 @@ def test_an_unreadable_outside_linked_worktree_keeps_the_mirror(self) -> None: why, bundles, _ = gd.archive_mirror(unit) finally: os.chmod(parent, 0o755) - self.assertIn("unreadable linked worktree", why) + self.assertIn("linked worktree", why) self.assertEqual(bundles, []) def test_a_commit_after_the_last_recheck_puts_the_mirror_back(self) -> None: