Skip to content

Commit e5aa178

Browse files
committed
Merge latest upstream/main into host completion fix
Signed-off-by: Duang777 <duangjl007@gmail.com>
2 parents 6b07f4d + 20dd1a3 commit e5aa178

4 files changed

Lines changed: 213 additions & 2 deletions

File tree

‎docs/architecture/rfcs/loopx-overall-roadmap-v0.md‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -930,6 +930,17 @@ provider. This is a bounded App companion, not full old-source/Host upgrade or
930930
D2/release-default qualification. Continue those original acceptance frontiers
931931
and retire each last caller separately.
932932

933+
Cold-import preservation checkpoint (R5/D1, T4/C1): full-state backups now
934+
witness each saved regular member's bytes in their existing manifests, including
935+
raw Markdown history, lease/receipt files, SQLite snapshots and stored
936+
configuration. Hashing the copied stream avoids binding a reviewed backup to a
937+
later source reread; failed reads retain the previous backup. This is the backup
938+
binding prerequisite, not cold-import qualification. Continue the same reviewed
939+
source import through inventory/quiescence, lease/outbox disposition, immutable
940+
preview, explicit confirm, writer fence, File/SQLite import/readback and original
941+
operation recovery on the installed CLI and packaged App. Do not retire the last
942+
normal Markdown writer or claim the released SQLite default from this checkpoint.
943+
933944
The qualified shadow→canonical archive→isolated File/SQLite restore path retains
934945
committed dependency archives, terminal lease facts and later canonical writes.
935946
It does not preserve every unreferenced old-source archive or raw historical

‎docs/reference/configuration-backup.md‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,31 @@ and [reviewed promotion](reviewed-coordination-promotion.md), then inspect the
8282
real Todo `authority_read` after cutover. An isolated SQLite archive or a
8383
configuration value alone cannot satisfy live provider acceptance.
8484

85+
## Physical file witnesses
86+
87+
Full `backup-state --execute` now records `execution.file_members` in both the
88+
in-archive manifest and the external manifest/readback. Each regular tar member
89+
has its archive path, copied byte count and SHA-256. Ordinary files are hashed
90+
while tarfile reads them; the witness never comes from rereading a source after
91+
it changed. SQLite members use the existing verified private snapshot, and
92+
`configuration-backup.json` witnesses its generated bytes. Unknown Markdown
93+
metadata, historical lease files and original receipts remain raw data.
94+
95+
Inspect the private external manifest with
96+
`jq '.execution.file_members' loopx-state-BACKUP_ID.manifest.json`. To validate an
97+
isolated restore, independently hash each listed regular member and compare its
98+
byte count. The list excludes the self-containing `manifest.json`, directories
99+
and tar links; retain the existing whole-archive checksum for their integrity
100+
and inspect link destinations separately. Earlier backups may lack this list;
101+
its absence does not grant migration qualification or erase their recovery
102+
support. A failed or short source read leaves an earlier backup intact.
103+
104+
These witnesses describe saved bytes, not one atomic observation of all live
105+
writers. A reviewed cold import still needs complete source discovery, writer
106+
and Host quiescence, lease/outbox disposition, immutable source/target binding,
107+
explicit confirmation and original-operation recovery. Neither this manifest
108+
nor configuration verification authorizes an import or a new execution grant.
109+
85110
## Privacy, consistency and limits
86111

87112
Backups are **private by default**. Full Goal rows can contain private paths,

‎loopx/state_backup.py‎

Lines changed: 44 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
import re
1010
import tarfile
1111
import tempfile
12-
from typing import Any
12+
from typing import Any, Protocol
1313

1414
from . import __version__
1515
from .paths import select_default_runtime_root
@@ -447,6 +447,45 @@ def build_state_backup_plan(
447447
}
448448

449449

450+
class _BackupReadStream(Protocol):
451+
def read(self, size: int = -1, /) -> bytes: ...
452+
453+
454+
class _BackupMemberReader:
455+
"""Witness the stream tarfile copies, without rereading a changing source."""
456+
457+
def __init__(self, source: _BackupReadStream) -> None:
458+
self.source = source
459+
self.digest = hashlib.sha256()
460+
self.size = 0
461+
462+
def read(self, size: int = -1, /) -> bytes:
463+
data = self.source.read(size)
464+
self.digest.update(data)
465+
self.size += len(data)
466+
return data
467+
468+
469+
class _BackupTarFile(tarfile.TarFile):
470+
"""Retain tarfile's link/metadata behavior and witness every copied member."""
471+
472+
def __init__(self, *args: Any, **kwargs: Any) -> None:
473+
self.file_members: list[dict[str, Any]] = []
474+
super().__init__(*args, **kwargs)
475+
476+
def addfile(self, tarinfo: tarfile.TarInfo, fileobj: _BackupReadStream | None = None) -> None:
477+
# The manifest contains this list; the whole-archive digest covers it.
478+
if not tarinfo.isfile() or tarinfo.name == "manifest.json":
479+
super().addfile(tarinfo, fileobj)
480+
return
481+
if fileobj is None:
482+
raise ValueError("backup regular member requires its copied byte stream")
483+
reader = _BackupMemberReader(fileobj)
484+
super().addfile(tarinfo, reader)
485+
self.file_members.append({"archive_path": tarinfo.name,
486+
"size_bytes": reader.size, "sha256": reader.digest.hexdigest()})
487+
488+
450489
def _add_path_to_tar(
451490
tar: tarfile.TarFile, source: Path, archive_path: str, exclude_roots: list[Path],
452491
staging: Path, snapshots: dict[Path, tuple[Path, dict[str, Any]]],
@@ -529,7 +568,7 @@ def execute_state_backup_plan(payload: dict[str, Any]) -> dict[str, Any]:
529568
staging = Path(temporary)
530569
staged_archive = staging / "archive.tar.gz"
531570
snapshots: dict[Path, tuple[Path, dict[str, Any]]] = {}
532-
with tarfile.open(staged_archive, "w:gz", dereference=False) as tar:
571+
with _BackupTarFile.open(staged_archive, "w:gz", dereference=False) as tar:
533572
for item in included:
534573
if not isinstance(item, dict):
535574
continue
@@ -550,6 +589,9 @@ def execute_state_backup_plan(payload: dict[str, Any]) -> dict[str, Any]:
550589
tar.addfile(info, io.BytesIO(configuration_bytes))
551590
updated["execution"]["configuration_backup"] = verify_configuration_backup(configuration)
552591
updated["execution"]["sqlite_snapshots"] = [entry[1] for entry in snapshots.values()]
592+
# manifest.json is deliberately excluded: it contains this list.
593+
# The existing external archive checksum witnesses the whole tar.
594+
updated["execution"]["file_members"] = tar.file_members
553595
manifest_bytes = json.dumps(updated, ensure_ascii=False, indent=2).encode("utf-8")
554596
info = tarfile.TarInfo("manifest.json")
555597
info.size = len(manifest_bytes)
Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
"""Backup witnesses describe archived bytes, never a later source reread."""
2+
from __future__ import annotations
3+
4+
import hashlib
5+
import json
6+
import os
7+
from pathlib import Path
8+
import subprocess
9+
import sqlite3
10+
import sys
11+
import tarfile
12+
13+
import pytest
14+
15+
from loopx.state_backup import build_state_backup_plan, execute_state_backup_plan
16+
17+
18+
def fixture(root: Path):
19+
project, runtime = root / "project", root / "runtime"
20+
state = project / ".codex/goals/example/ACTIVE_GOAL_STATE.md"
21+
state.parent.mkdir(parents=True)
22+
state.write_bytes(b"## Agent Todo\n- [ ] Current work\n## Agent Todo Archive\n"
23+
b"- [x] Unreferenced history\n <!-- future_metadata=null -->\n")
24+
registry = project / ".loopx/registry.json"
25+
registry.parent.mkdir(parents=True)
26+
registry.write_text(json.dumps({"common_runtime_root": str(runtime), "goals": [
27+
{"id": "example", "repo": str(project), "state_file": str(state),
28+
"future_configuration": {"disabled": False, "unset": None}}]}))
29+
historical = runtime / "goals/example/task-leases/retained.json"
30+
historical.parent.mkdir(parents=True)
31+
historical.write_bytes(b'{"status":"released","future":{"null":null,"flag":false}}\n')
32+
receipt = runtime / "goals/example/receipts/original.json"
33+
receipt.parent.mkdir(parents=True)
34+
receipt.write_bytes(b'{"operation_id":"original","unknown":[false,null]}\n')
35+
(runtime / "receipt-link").symlink_to("goals/example/receipts/original.json")
36+
os.link(receipt, runtime / "receipt-hardlink")
37+
return project, runtime, state, registry
38+
39+
40+
def plan(root: Path):
41+
project, runtime, state, registry = fixture(root)
42+
return build_state_backup_plan(project=project, runtime_root=runtime,
43+
output_dir=root / "backups", backup_id="fixture", include_automations=False,
44+
include_skills=False, include_registry_projects=False, registry_path=registry), state
45+
46+
47+
def assert_members(payload):
48+
members = payload["execution"]["file_members"]
49+
by_path = {row["archive_path"]: row for row in members}
50+
assert len(by_path) == len(members)
51+
with tarfile.open(payload["archive_path"]) as archive:
52+
expected = {row.name for row in archive if row.isfile() and row.name != "manifest.json"}
53+
assert set(by_path) == expected
54+
for path, row in by_path.items():
55+
data = archive.extractfile(path).read()
56+
assert row == {"archive_path": path, "size_bytes": len(data),
57+
"sha256": hashlib.sha256(data).hexdigest()}
58+
internal = json.load(archive.extractfile("manifest.json"))
59+
assert internal["execution"]["file_members"] == members
60+
external = json.loads(Path(payload["manifest_path"]).read_text())
61+
assert external["execution"]["file_members"] == members
62+
63+
64+
@pytest.mark.parametrize("with_sqlite", [False, True])
65+
def test_real_cli_inventory_binds_raw_history_and_generated_configuration(tmp_path, with_sqlite):
66+
project, runtime, state, registry = fixture(tmp_path)
67+
original = state.read_bytes()
68+
if with_sqlite:
69+
with sqlite3.connect(runtime / "retained.data") as database:
70+
database.execute("CREATE TABLE receipts(value TEXT)")
71+
database.execute("INSERT INTO receipts VALUES(?)", ('{"future":[false,null]}',))
72+
result = subprocess.run([sys.executable, "-I", "-m", "loopx.cli", "--registry", str(registry),
73+
"--runtime-root", str(runtime), "--format", "json", "backup-state", "--project", str(project),
74+
"--output-dir", str(tmp_path / "backups"), "--current-project-only", "--no-automations",
75+
"--no-skills", "--execute"], env={**os.environ, "LOOPX_USAGE_PING": "0"},
76+
capture_output=True, text=True, check=True)
77+
payload = json.loads(result.stdout)
78+
assert payload["ok"] is True
79+
assert_members(payload)
80+
restored = tmp_path / "restored"
81+
with tarfile.open(payload["archive_path"]) as archive:
82+
archive.extractall(restored, filter="data")
83+
assert (restored / "project/.codex/goals/example/ACTIVE_GOAL_STATE.md").read_bytes() == original
84+
assert (restored / "runtime-root/receipt-link").is_symlink()
85+
assert (restored / "runtime-root/receipt-hardlink").read_bytes() == (
86+
runtime / "goals/example/receipts/original.json").read_bytes()
87+
if with_sqlite:
88+
with sqlite3.connect(restored / "runtime-root/retained.data") as database:
89+
assert database.execute("PRAGMA integrity_check").fetchone()[0] == "ok"
90+
assert database.execute("SELECT value FROM receipts").fetchone()[0] == '{"future":[false,null]}'
91+
assert state.read_bytes() == original
92+
93+
94+
def test_member_digest_uses_the_copied_stream_not_a_later_source_reread(tmp_path, monkeypatch):
95+
payload, state = plan(tmp_path)
96+
original = state.read_bytes()
97+
addfile = tarfile.TarFile.addfile
98+
99+
def change_after_copy(archive, info, fileobj=None):
100+
result = addfile(archive, info, fileobj)
101+
if info.name == "project/.codex/goals/example/ACTIVE_GOAL_STATE.md":
102+
state.write_bytes(b"later source contents\n")
103+
return result
104+
105+
monkeypatch.setattr(tarfile.TarFile, "addfile", change_after_copy)
106+
completed = execute_state_backup_plan(payload)
107+
assert_members(completed)
108+
row = next(row for row in completed["execution"]["file_members"]
109+
if row["archive_path"] == "project/.codex/goals/example/ACTIVE_GOAL_STATE.md")
110+
assert row["sha256"] == hashlib.sha256(original).hexdigest()
111+
assert row["sha256"] != hashlib.sha256(state.read_bytes()).hexdigest()
112+
113+
114+
def test_short_source_read_cannot_publish_partial_inventory_or_replace_backup(tmp_path, monkeypatch):
115+
payload, state = plan(tmp_path)
116+
output = Path(payload["output_dir"])
117+
output.mkdir()
118+
archive, manifest = Path(payload["archive_path"]), Path(payload["manifest_path"])
119+
archive.write_bytes(b"previous archive")
120+
manifest.write_bytes(b"previous manifest")
121+
addfile = tarfile.TarFile.addfile
122+
123+
def truncate_before_copy(archive, info, fileobj=None):
124+
if info.name == "project/.codex/goals/example/ACTIVE_GOAL_STATE.md":
125+
state.write_bytes(b"")
126+
return addfile(archive, info, fileobj)
127+
128+
monkeypatch.setattr(tarfile.TarFile, "addfile", truncate_before_copy)
129+
with pytest.raises(OSError, match="unexpected end of data"):
130+
execute_state_backup_plan(payload)
131+
assert archive.read_bytes() == b"previous archive"
132+
assert manifest.read_bytes() == b"previous manifest"
133+
assert sorted(path.name for path in output.iterdir()) == sorted([archive.name, manifest.name])

0 commit comments

Comments
 (0)