From 29c3f4ed6150ac2795cf3f2b8d2fec46badf4d5e Mon Sep 17 00:00:00 2001 From: OpenClaw Agent Date: Thu, 16 Apr 2026 23:09:58 -0700 Subject: [PATCH 1/8] feat(reproduce): add list_experiments.py inventory tool Scans autonomous-learning-library/{examples,all/scripts,all/r3,all/presets} and emits a per-category Markdown (or --json) table with paths, default frame counts, and env/agent hints. Pure stdlib so it runs on the Jetson Orin venv without extra deps. This is the first piece of the new reproduce/ folder that documents how to reproduce the RTSS 2023 R^3 paper on the modernized PyTorch 2.8 + Gymnasium stack. --- reproduce/list_experiments.py | 250 ++++++++++++++++++++++++++++++++++ 1 file changed, 250 insertions(+) create mode 100644 reproduce/list_experiments.py diff --git a/reproduce/list_experiments.py b/reproduce/list_experiments.py new file mode 100644 index 0000000..802cea5 --- /dev/null +++ b/reproduce/list_experiments.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +""" +list_experiments.py — Inventory R³ paper reproduction targets. + +Scans the autonomous-learning-library (ALL) sub-tree for: + + 1. ``examples/experiment*.py`` — hand-written ALL example drivers. + 2. ``all/scripts/train_*.py`` — installed CLI entry points (``all-classic``, + ``all-atari``, ``all-continuous`` …). + 3. ``all/r3/*.py`` (excluding tests) — the R³ runtime modules that are wired + into the agents. + 4. ``all/presets//.py`` — the agent presets (DQN, DDQN, PPO, + Rainbow …) that the train scripts can + launch. + +For each scanned file we extract the leading docstring (if any), the default +``--frames`` from any ``argparse`` declaration, and any obvious environment / +agent hints. Everything is printed as a Markdown table to ``stdout`` so it can +be redirected into ``reproduce/EXPERIMENTS.md`` or pasted into a GitHub PR. + +Usage +----- + + python reproduce/list_experiments.py # full markdown report + python reproduce/list_experiments.py --json # raw JSON instead + +The script intentionally has no third-party dependencies (only stdlib) so it +runs on the Jetson Orin venv and CI alike. +""" + +from __future__ import annotations + +import argparse +import ast +import json +import os +import re +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Iterable, List, Optional + + +REPO_ROOT = Path(__file__).resolve().parent.parent +ALL_ROOT = REPO_ROOT / "autonomous-learning-library" + + +@dataclass +class ScriptInfo: + relpath: str + category: str + docstring: str = "" + default_frames: Optional[str] = None + envs: List[str] = field(default_factory=list) + agents: List[str] = field(default_factory=list) + notes: str = "" + + def to_dict(self) -> dict: + return { + "relpath": self.relpath, + "category": self.category, + "docstring": self.docstring, + "default_frames": self.default_frames, + "envs": self.envs, + "agents": self.agents, + "notes": self.notes, + } + + +def _read_text(path: Path) -> str: + try: + return path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + return "" + + +def _module_docstring(path: Path) -> str: + src = _read_text(path) + if not src: + return "" + try: + tree = ast.parse(src) + except SyntaxError: + return "" + doc = ast.get_docstring(tree) or "" + # Keep only the first non-empty line — the table cell stays compact. + for line in doc.splitlines(): + line = line.strip() + if line: + return line + return "" + + +_FRAMES_RE = re.compile( + r"""['"]--frames['"][^)]*?default\s*=\s*([^,)\s]+)""", + re.DOTALL, +) +_FRAMES_KW_RE = re.compile( + r"""default_frames\s*=\s*([^,)\s]+)""", + re.DOTALL, +) +_ENV_HINTS_RE = re.compile( + r"""(?:CartPole-v[01]|Acrobot-v1|MountainCar(?:Continuous)?-v0|""" + r"""LunarLander(?:Continuous)?-v[23]|PongNoFrameskip-v4|BreakoutNoFrameskip-v4|""" + r"""SeaquestNoFrameskip-v4|Pendulum-v1|HalfCheetah-v[24]|Hopper-v[24]|Ant-v[24])""", +) +_AGENT_HINTS_RE = re.compile( + r"""\b(dqn|ddqn|c51|rainbow|a2c|ppo|vac|vpg|vqn|vsarsa|sac|ddpg|td3)\b""", + re.IGNORECASE, +) + + +def _scan_argparse(path: Path) -> Optional[str]: + src = _read_text(path) + if not src: + return None + m = _FRAMES_RE.search(src) + if m: + return m.group(1).strip() + m = _FRAMES_KW_RE.search(src) + if m: + return m.group(1).strip() + return None + + +def _scan_hints(path: Path) -> tuple[List[str], List[str]]: + src = _read_text(path) + if not src: + return ([], []) + envs = sorted(set(_ENV_HINTS_RE.findall(src))) + agents = sorted({a.lower() for a in _AGENT_HINTS_RE.findall(src)}) + return envs, agents + + +def _iter_scripts() -> Iterable[ScriptInfo]: + if not ALL_ROOT.exists(): + return + + # 1. examples/experiment*.py + examples_dir = ALL_ROOT / "examples" + if examples_dir.exists(): + for p in sorted(examples_dir.glob("experiment*.py")): + envs, agents = _scan_hints(p) + yield ScriptInfo( + relpath=str(p.relative_to(REPO_ROOT)), + category="example", + docstring=_module_docstring(p), + default_frames=_scan_argparse(p), + envs=envs, + agents=agents, + ) + + # 2. all/scripts/train_*.py + scripts_dir = ALL_ROOT / "all" / "scripts" + if scripts_dir.exists(): + for p in sorted(scripts_dir.glob("train_*.py")): + envs, agents = _scan_hints(p) + yield ScriptInfo( + relpath=str(p.relative_to(REPO_ROOT)), + category="cli-entry", + docstring=_module_docstring(p), + default_frames=_scan_argparse(p), + envs=envs, + agents=agents, + notes=f"console_script: all-{p.stem.replace('train_', '')}", + ) + + # 3. all/r3/*.py (skip tests) + r3_dir = ALL_ROOT / "all" / "r3" + if r3_dir.exists(): + for p in sorted(r3_dir.glob("*.py")): + if p.name.endswith("_test.py") or p.name == "__init__.py": + continue + yield ScriptInfo( + relpath=str(p.relative_to(REPO_ROOT)), + category="r3-module", + docstring=_module_docstring(p), + ) + + # 4. agent presets + presets_root = ALL_ROOT / "all" / "presets" + if presets_root.exists(): + for domain_dir in sorted(p for p in presets_root.iterdir() if p.is_dir()): + for p in sorted(domain_dir.glob("*.py")): + if p.name == "__init__.py": + continue + yield ScriptInfo( + relpath=str(p.relative_to(REPO_ROOT)), + category=f"preset:{domain_dir.name}", + docstring=_module_docstring(p), + agents=[p.stem], + ) + + +def _md_escape(s: str) -> str: + return s.replace("|", "\\|") + + +def _render_markdown(scripts: List[ScriptInfo]) -> str: + lines: List[str] = [] + lines.append("# R³ paper-reproduction script inventory") + lines.append("") + lines.append(f"Generated by `reproduce/list_experiments.py` over `{ALL_ROOT.relative_to(REPO_ROOT)}`.") + lines.append("") + + # Group by category for readability. + by_cat: dict[str, List[ScriptInfo]] = {} + for s in scripts: + by_cat.setdefault(s.category, []).append(s) + + for cat in sorted(by_cat): + lines.append(f"## {cat}") + lines.append("") + lines.append("| Path | Default frames | Env hints | Agent hints | Description |") + lines.append("|------|----------------|-----------|-------------|-------------|") + for s in by_cat[cat]: + desc = s.docstring or s.notes or "" + lines.append( + "| `{path}` | {frames} | {envs} | {agents} | {desc} |".format( + path=_md_escape(s.relpath), + frames=_md_escape(s.default_frames or "—"), + envs=_md_escape(", ".join(s.envs) or "—"), + agents=_md_escape(", ".join(s.agents) or "—"), + desc=_md_escape(desc), + ) + ) + lines.append("") + return "\n".join(lines) + + +def main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.strip().splitlines()[0]) + parser.add_argument("--json", action="store_true", help="emit JSON instead of Markdown") + args = parser.parse_args(argv) + + if not ALL_ROOT.exists(): + print(f"error: cannot find {ALL_ROOT}", file=sys.stderr) + return 2 + + scripts = list(_iter_scripts()) + if args.json: + json.dump([s.to_dict() for s in scripts], sys.stdout, indent=2) + sys.stdout.write("\n") + else: + sys.stdout.write(_render_markdown(scripts)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 51e190b8f1907dab91cc3c08d544cd8d410671d7 Mon Sep 17 00:00:00 2001 From: OpenClaw Agent Date: Thu, 16 Apr 2026 23:10:09 -0700 Subject: [PATCH 2/8] feat(reproduce): add CartPole classic-control smoke test run_smoke_classic.sh exercises the modernized ALL stack on CartPole-v1 with DQN for a tiny number of frames (default 5000). Used as a fallback sanity gate when the Atari ROMs are not yet installed on the host. Writes everything under /experiment/zexin/R3/reproduce/runs/ to respect the Jetson Orin disk-layout rule (NVMe only). --- reproduce/run_smoke_classic.sh | 161 +++++++++++++++++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100755 reproduce/run_smoke_classic.sh diff --git a/reproduce/run_smoke_classic.sh b/reproduce/run_smoke_classic.sh new file mode 100755 index 0000000..7441308 --- /dev/null +++ b/reproduce/run_smoke_classic.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +# reproduce/run_smoke.sh — 5–10 minute end-to-end smoke test for the R³ stack. +# +# What this script does +# --------------------- +# 1. Runs the R³ algorithmic unit tests (replay_buffer_size, +# batch_size_control, deadline, runtime_coordinator) — these are pure-Python +# and should finish in seconds. They guard the R³ core math. +# 2. Trains a tiny DQN on CartPole-v1 for a very small number of frames +# (default: 5 000) using the installed ``all-classic`` CLI. This exercises +# the full ALL pipeline (env → preset → agent → replay buffer → optimizer +# → tensorboard logging) on real hardware. +# 3. Parses the resulting tensorboard log directory under the run dir and +# prints the final ``test/returns/mean`` and ``test/returns/std`` to +# stdout, then exits 0 (PASS) or non-zero (FAIL). +# +# Output layout (NVMe only — never the eMMC root) +# ----------------------------------------------- +# /experiment/zexin/R3/reproduce/runs/smoke_/ +# runs/ ← ALL tensorboard subdir +# smoke.log ← captured stdout/stderr +# summary.json ← parsed metrics +# +# Usage +# ----- +# bash reproduce/run_smoke.sh # default 5000 frames +# FRAMES=10000 bash reproduce/run_smoke.sh # override +# AGENT=dqn ENV=CartPole-v1 bash reproduce/run_smoke.sh +# +# Exit codes +# ---------- +# 0 PASS — both unit tests and the mini training run completed without errors. +# 1 FAIL — unit tests reported a failure. +# 2 FAIL — training process crashed. +# 3 FAIL — could not parse the tensorboard log to recover any metric. +# +set -u +set -o pipefail + +#------------------------------------------------------------------------------# +# Knobs (override via env vars). +#------------------------------------------------------------------------------# +FRAMES="${FRAMES:-5000}" +ENV_NAME="${ENV:-CartPole-v1}" +AGENT_NAME="${AGENT:-dqn}" +DEVICE="${DEVICE:-cuda}" +RUNS_ROOT="${RUNS_ROOT:-/experiment/zexin/R3/reproduce/runs}" +TS="$(date -u +%Y%m%d_%H%M%SZ)" +RUN_DIR="${RUNS_ROOT}/smoke_${TS}" +LOG_FILE="${RUN_DIR}/smoke.log" + +#------------------------------------------------------------------------------# +# Locate repo root (this script lives in /reproduce/). +#------------------------------------------------------------------------------# +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +mkdir -p "${RUN_DIR}" +echo "[smoke] repo: ${REPO_ROOT}" +echo "[smoke] run_dir: ${RUN_DIR}" +echo "[smoke] env=${ENV_NAME} agent=${AGENT_NAME} device=${DEVICE} frames=${FRAMES}" +echo "[smoke] log: ${LOG_FILE}" +echo + +#------------------------------------------------------------------------------# +# Step 1 — R³ unit tests (pure CPU, seconds). +#------------------------------------------------------------------------------# +echo "[smoke] step 1/3 — R³ algorithmic unit tests" +{ + cd "${REPO_ROOT}/autonomous-learning-library" + python -m unittest discover -s all/r3 -p "*_test.py" -v +} 2>&1 | tee -a "${LOG_FILE}" +unit_rc="${PIPESTATUS[0]}" +if [ "${unit_rc}" -ne 0 ]; then + echo "[smoke] FAIL: R³ unit tests exited ${unit_rc}" | tee -a "${LOG_FILE}" + exit 1 +fi + +#------------------------------------------------------------------------------# +# Step 2 — micro training run via all-classic. +#------------------------------------------------------------------------------# +echo +echo "[smoke] step 2/3 — micro training: ${AGENT_NAME} on ${ENV_NAME} for ${FRAMES} frames" +mkdir -p "${RUN_DIR}/runs" +{ + cd "${RUN_DIR}" + # ``all-classic`` is installed by autonomous-learning-library's setup.py. + # We force a small replay-buffer hyperparameter so this fits comfortably + # in Orin RAM and finishes well under 10 minutes even on CPU. + all-classic "${ENV_NAME}" "${AGENT_NAME}" \ + --device "${DEVICE}" \ + --frames "${FRAMES}" \ + --logdir "${RUN_DIR}/runs" \ + --hyperparameters replay_buffer_size=2000 +} 2>&1 | tee -a "${LOG_FILE}" +train_rc="${PIPESTATUS[0]}" +if [ "${train_rc}" -ne 0 ]; then + echo "[smoke] FAIL: training exited ${train_rc}" | tee -a "${LOG_FILE}" + exit 2 +fi + +#------------------------------------------------------------------------------# +# Step 3 — parse tensorboard logs and print final metric. +#------------------------------------------------------------------------------# +echo +echo "[smoke] step 3/3 — parsing test return metrics" +export RUN_DIR +python - <<'PY' 2>&1 | tee -a "${LOG_FILE}" +import json +import os +import sys +from pathlib import Path + +run_dir = Path(os.environ["RUN_DIR"]) / "runs" +event_files = sorted(run_dir.rglob("events.out.tfevents.*")) +if not event_files: + print(f"[smoke] FAIL: no tensorboard event files under {run_dir}", file=sys.stderr) + sys.exit(3) + +try: + from tensorboard.backend.event_processing.event_accumulator import EventAccumulator +except Exception as e: + print(f"[smoke] WARN: tensorboard import failed ({e}); cannot parse metrics." + f"\n[smoke] Smoke training itself succeeded — treating as PASS.") + sys.exit(0) + +best = {"path": None, "tag": None, "value": None, "step": None} +for ef in event_files: + acc = EventAccumulator(str(ef)) + try: + acc.Reload() + except Exception as e: + print(f"[smoke] note: failed to load {ef}: {e}") + continue + tags = acc.Tags().get("scalars", []) + # ALL writes scalars like "evaluation/returns/mean" or "test/returns/mean". + for tag in tags: + if tag.endswith("returns/mean") and ("test" in tag or "evaluation" in tag): + scalars = acc.Scalars(tag) + if scalars: + last = scalars[-1] + if best["value"] is None or last.step > (best["step"] or -1): + best = {"path": str(ef), "tag": tag, "value": float(last.value), "step": int(last.step)} + +summary_path = Path(os.environ["RUN_DIR"]) / "summary.json" +summary_path.write_text(json.dumps(best, indent=2)) +print(f"[smoke] summary -> {summary_path}") +print(json.dumps(best, indent=2)) + +if best["value"] is None: + print("[smoke] WARN: did not find a test/returns/mean tag; pipeline still ran cleanly.") +PY +parse_rc="${PIPESTATUS[0]}" +if [ "${parse_rc}" -ne 0 ]; then + echo "[smoke] FAIL: metric parser exited ${parse_rc}" | tee -a "${LOG_FILE}" + exit 3 +fi + +echo +echo "[smoke] ✅ PASS — see ${RUN_DIR}" +exit 0 From 8f29cf11b88f58670149ee7acb15c07bfa19171a Mon Sep 17 00:00:00 2001 From: OpenClaw Agent Date: Thu, 16 Apr 2026 23:10:09 -0700 Subject: [PATCH 3/8] feat(reproduce): add Atari Pong smoke test (primary 5-10 min gate) run_smoke.sh is the canonical smoke gate for the R^3 reproduction: 1. Runs all R^3 algorithmic unit tests (deadline, batch_size_control, runtime_coordinator, replay_buffer_size). 2. Preflights ale_py + Pong ROM availability with actionable hints if missing. 3. Trains DQN on PongNoFrameskip-v4 for FRAMES (default 20000) frames via the installed all-atari CLI. 4. Parses tensorboard logs and prints final returns/mean to stdout. Exit codes (0 PASS, 1 unit-test fail, 2 ALE missing, 3 train crash, 4 log-parse fail) make it suitable as an automated CI gate. Output goes to /experiment/zexin/R3/reproduce/runs/smoke_/ on the NVMe. --- reproduce/run_smoke.sh | 210 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100755 reproduce/run_smoke.sh diff --git a/reproduce/run_smoke.sh b/reproduce/run_smoke.sh new file mode 100755 index 0000000..e4fe62f --- /dev/null +++ b/reproduce/run_smoke.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env bash +# reproduce/run_smoke.sh — 5–10 minute Atari smoke test for the R³ stack. +# +# Goal +# ---- +# Verify end-to-end that the modernized R³ stack can actually train a DQN agent +# on the canonical Atari benchmark (PongNoFrameskip-v4) on a Jetson Orin (or +# any CUDA host). This is the *Atari* smoke test — the classic-control +# CartPole variant lives in ``run_smoke_classic.sh``. +# +# Why Pong? +# - R³ (RTSS 2023) explicitly evaluates on Atari (see paper §VI). Pong is the +# fastest-converging classical Atari benchmark and is the standard "does +# the pipeline work?" gate for any DQN-family implementation. +# - On Orin AGX, ~10–20k frames of DQN-Pong is enough to confirm the env +# wrapper, replay buffer, optimizer, and tensorboard logging all wire up +# correctly without waiting for actual learning. +# +# What this script does +# --------------------- +# 1. Run the R³ algorithmic unit tests (pure Python, seconds). +# 2. Sanity-check that ``ale_py`` is importable and at least one Atari ROM +# is registered. If not, print actionable install hints and FAIL fast. +# 3. Train ``dqn`` on ``PongNoFrameskip-v4`` for FRAMES (default 20 000) +# via the installed ``all-atari`` CLI, with a tiny replay buffer to keep +# RAM bounded. +# 4. Parse the resulting tensorboard log and print the final +# ``test/returns/mean`` / ``returns/mean`` to stdout. +# +# Output layout (NVMe only — never the eMMC root) +# ----------------------------------------------- +# /experiment/zexin/R3/reproduce/runs/smoke_/ +# runs/ ← ALL tensorboard subdir +# smoke.log ← captured stdout/stderr +# summary.json ← parsed metrics (best-effort) +# +# Usage +# ----- +# bash reproduce/run_smoke.sh # default 20000 frames +# FRAMES=10000 bash reproduce/run_smoke.sh # smaller / faster +# ENV=BreakoutNoFrameskip-v4 bash reproduce/run_smoke.sh +# +# Exit codes +# ---------- +# 0 PASS — unit tests + Atari training + log parsing all clean. +# 1 FAIL — R³ unit tests failed. +# 2 FAIL — Atari runtime not available (ale_py / ROMs missing). +# 3 FAIL — training crashed. +# 4 FAIL — could not parse tensorboard logs. +# +set -u +set -o pipefail + +#------------------------------------------------------------------------------# +# Knobs (override via env vars). +#------------------------------------------------------------------------------# +FRAMES="${FRAMES:-20000}" +ENV_NAME="${ENV:-PongNoFrameskip-v4}" +AGENT_NAME="${AGENT:-dqn}" +DEVICE="${DEVICE:-cuda}" +RUNS_ROOT="${RUNS_ROOT:-/experiment/zexin/R3/reproduce/runs}" +TS="$(date -u +%Y%m%d_%H%M%SZ)" +RUN_DIR="${RUNS_ROOT}/smoke_${TS}" +LOG_FILE="${RUN_DIR}/smoke.log" + +#------------------------------------------------------------------------------# +# Locate repo root (this script lives in /reproduce/). +#------------------------------------------------------------------------------# +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +mkdir -p "${RUN_DIR}/runs" +echo "[smoke] repo: ${REPO_ROOT}" +echo "[smoke] run_dir: ${RUN_DIR}" +echo "[smoke] env=${ENV_NAME} agent=${AGENT_NAME} device=${DEVICE} frames=${FRAMES}" +echo "[smoke] log: ${LOG_FILE}" +echo + +#------------------------------------------------------------------------------# +# Step 1 — R³ unit tests (pure CPU, seconds). +#------------------------------------------------------------------------------# +echo "[smoke] step 1/4 — R³ algorithmic unit tests" +{ + cd "${REPO_ROOT}/autonomous-learning-library" + python -m unittest discover -s all/r3 -p "*_test.py" -v +} 2>&1 | tee -a "${LOG_FILE}" +unit_rc="${PIPESTATUS[0]}" +if [ "${unit_rc}" -ne 0 ]; then + echo "[smoke] FAIL: R³ unit tests exited ${unit_rc}" | tee -a "${LOG_FILE}" + exit 1 +fi + +#------------------------------------------------------------------------------# +# Step 2 — Atari runtime preflight. +#------------------------------------------------------------------------------# +echo +echo "[smoke] step 2/4 — Atari runtime preflight (ale_py + ROM availability)" +python - <<'PY' 2>&1 | tee -a "${LOG_FILE}" +import sys +try: + import ale_py # noqa: F401 + print(f"[preflight] ale_py {ale_py.__version__}") +except Exception as e: + print(f"[preflight] FAIL: cannot import ale_py: {e}") + print("[preflight] hint: pip install 'ale-py' 'autorom[accept-rom-license]' && AutoROM --accept-license") + sys.exit(2) + +try: + import gymnasium as gym + gym.register_envs(ale_py) + env = gym.make("PongNoFrameskip-v4") + env.reset() + env.close() + print("[preflight] PongNoFrameskip-v4 constructible — ROM available.") +except Exception as e: + print(f"[preflight] FAIL: cannot make PongNoFrameskip-v4: {e}") + print("[preflight] hint: AutoROM --accept-license # downloads the Atari 2600 ROM set") + sys.exit(2) +PY +pre_rc="${PIPESTATUS[0]}" +if [ "${pre_rc}" -ne 0 ]; then + echo "[smoke] FAIL: Atari runtime not ready (rc=${pre_rc}). See hints above." | tee -a "${LOG_FILE}" + exit 2 +fi + +#------------------------------------------------------------------------------# +# Step 3 — micro Atari training run via all-atari. +#------------------------------------------------------------------------------# +echo +echo "[smoke] step 3/4 — micro Atari training: ${AGENT_NAME} on ${ENV_NAME} for ${FRAMES} frames" +{ + cd "${RUN_DIR}" + # Tiny replay buffer to keep Orin RAM happy at smoke scale; full paper + # config uses 1e6 frames buffer and is exercised by run_short / run_full. + all-atari "${ENV_NAME}" "${AGENT_NAME}" \ + --device "${DEVICE}" \ + --frames "${FRAMES}" \ + --logdir "${RUN_DIR}/runs" \ + --hyperparameters replay_buffer_size=10000 replay_start_size=1000 +} 2>&1 | tee -a "${LOG_FILE}" +train_rc="${PIPESTATUS[0]}" +if [ "${train_rc}" -ne 0 ]; then + echo "[smoke] FAIL: training exited ${train_rc}" | tee -a "${LOG_FILE}" + exit 3 +fi + +#------------------------------------------------------------------------------# +# Step 4 — parse tensorboard logs. +#------------------------------------------------------------------------------# +echo +echo "[smoke] step 4/4 — parsing test return metrics" +export RUN_DIR +python - <<'PY' 2>&1 | tee -a "${LOG_FILE}" +import json +import os +import sys +from pathlib import Path + +run_dir = Path(os.environ["RUN_DIR"]) / "runs" +event_files = sorted(run_dir.rglob("events.out.tfevents.*")) +if not event_files: + print(f"[smoke] FAIL: no tensorboard event files under {run_dir}", file=sys.stderr) + sys.exit(4) + +try: + from tensorboard.backend.event_processing.event_accumulator import EventAccumulator +except Exception as e: + print(f"[smoke] WARN: tensorboard import failed ({e}); cannot parse metrics." + f"\n[smoke] Smoke training itself succeeded — treating as PASS.") + sys.exit(0) + +best = {"path": None, "tag": None, "value": None, "step": None} +for ef in event_files: + acc = EventAccumulator(str(ef)) + try: + acc.Reload() + except Exception as e: + print(f"[smoke] note: failed to load {ef}: {e}") + continue + tags = acc.Tags().get("scalars", []) + for tag in tags: + if tag.endswith("returns/mean") or tag.endswith("returns_100/mean"): + scalars = acc.Scalars(tag) + if scalars: + last = scalars[-1] + if best["value"] is None or last.step > (best["step"] or -1): + best = { + "path": str(ef), + "tag": tag, + "value": float(last.value), + "step": int(last.step), + } + +summary_path = Path(os.environ["RUN_DIR"]) / "summary.json" +summary_path.write_text(json.dumps(best, indent=2)) +print(f"[smoke] summary -> {summary_path}") +print(json.dumps(best, indent=2)) + +if best["value"] is None: + print("[smoke] WARN: no returns/mean tag found — pipeline still ran cleanly.") +PY +parse_rc="${PIPESTATUS[0]}" +if [ "${parse_rc}" -ne 0 ]; then + echo "[smoke] FAIL: metric parser exited ${parse_rc}" | tee -a "${LOG_FILE}" + exit 4 +fi + +echo +echo "[smoke] ✅ PASS — see ${RUN_DIR}" +exit 0 From 9ab9164526b5daad5651d7b4e02604cc14fee2be Mon Sep 17 00:00:00 2001 From: OpenClaw Agent Date: Thu, 16 Apr 2026 23:10:23 -0700 Subject: [PATCH 4/8] feat(reproduce): add 30-60 min Atari short paper-validation script run_short.sh trains DQN on PongNoFrameskip-v4 for 250000 frames and is the bridge between the smoke gate and a full paper run. Designed to be detached from the user's shell: - Recommended invocation is via tmux (see header docstring). - --background flag self-daemonizes via setsid+nohup as a fallback when tmux is not available. - Maintains a short_LATEST symlink under reproduce/runs/ so the user can always 'tail -F short_LATEST/short.log'. - Emits progress.json with all parsed returns/mean snapshots at the end. Documented inspection commands include 'tmux attach -t r3-short' and 'tensorboard --logdir reproduce/runs/short_LATEST/runs --bind_all'. --- reproduce/run_short.sh | 178 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100755 reproduce/run_short.sh diff --git a/reproduce/run_short.sh b/reproduce/run_short.sh new file mode 100755 index 0000000..a0c2e33 --- /dev/null +++ b/reproduce/run_short.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# reproduce/run_short.sh — 30–60 minute "short paper-validation" Atari run. +# +# Goal +# ---- +# Train DQN on PongNoFrameskip-v4 for enough frames (default 250 000) that we +# expect to see early signs of learning (returns climbing out of the random +# baseline of ≈-21 toward ≈-15..-10) on a Jetson Orin AGX. This is *not* a +# full paper reproduction — that lives in ``run_full_paper.sh``. It IS a strong +# smoke test that the modernized R³ stack converges in the same direction as +# the published RTSS 2023 numbers. +# +# Designed to run fully detached +# ------------------------------ +# The recommended invocation is: +# +# tmux new-session -d -s r3-short \ +# "source /experiment/zexin/venvs/r3/bin/activate && \ +# cd /experiment/zexin/R3 && \ +# bash reproduce/run_short.sh 2>&1" +# +# The script itself also accepts ``--background`` which double-forks via +# ``setsid`` + ``nohup``, so it survives an SSH disconnect even without tmux. +# +# Output layout (NVMe only — never the eMMC root) +# ----------------------------------------------- +# /experiment/zexin/R3/reproduce/runs/short_/ +# runs/ ← ALL tensorboard subdir +# short.log ← captured stdout/stderr +# progress.json ← parsed periodic snapshots (best-effort) +# LATEST -> ... ← convenience symlink the user can tail +# +# Resume / inspect +# ---------------- +# tmux attach -t r3-short +# tail -F /experiment/zexin/R3/reproduce/runs/short_*/short.log | head +# tensorboard --logdir /experiment/zexin/R3/reproduce/runs/short_/runs +# +set -u +set -o pipefail + +#------------------------------------------------------------------------------# +# Knobs. +#------------------------------------------------------------------------------# +FRAMES="${FRAMES:-250000}" +ENV_NAME="${ENV:-PongNoFrameskip-v4}" +AGENT_NAME="${AGENT:-dqn}" +DEVICE="${DEVICE:-cuda}" +RUNS_ROOT="${RUNS_ROOT:-/experiment/zexin/R3/reproduce/runs}" +TS="$(date -u +%Y%m%d_%H%M%SZ)" +RUN_DIR="${RUNS_ROOT}/short_${TS}" +LOG_FILE="${RUN_DIR}/short.log" +LATEST_LINK="${RUNS_ROOT}/short_LATEST" + +#------------------------------------------------------------------------------# +# Optional --background self-daemonize (when tmux is not used). +#------------------------------------------------------------------------------# +if [ "${1:-}" = "--background" ]; then + shift + mkdir -p "${RUN_DIR}" + echo "[short] re-launching detached; logs -> ${LOG_FILE}" + setsid nohup bash "$0" "$@" >"${LOG_FILE}" 2>&1 < /dev/null & + echo "[short] background pid: $!" + exit 0 +fi + +#------------------------------------------------------------------------------# +# Locate repo root. +#------------------------------------------------------------------------------# +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +mkdir -p "${RUN_DIR}/runs" +ln -sfn "${RUN_DIR}" "${LATEST_LINK}" || true + +echo "[short] repo: ${REPO_ROOT}" +echo "[short] run_dir: ${RUN_DIR}" +echo "[short] env=${ENV_NAME} agent=${AGENT_NAME} device=${DEVICE} frames=${FRAMES}" +echo "[short] log: ${LOG_FILE}" +echo "[short] symlink: ${LATEST_LINK} -> ${RUN_DIR}" +echo "[short] progress: tail -F ${LOG_FILE}" +echo + +#------------------------------------------------------------------------------# +# Quick preflight (don't re-run the full unit suite — smoke owns that). +#------------------------------------------------------------------------------# +python - <<'PY' 2>&1 | tee -a "${LOG_FILE}" +import sys +try: + import ale_py, gymnasium as gym + gym.register_envs(ale_py) + env = gym.make("PongNoFrameskip-v4") + env.reset(); env.close() + print("[short] preflight: ale_py + Pong ROM OK") +except Exception as e: + print(f"[short] preflight FAIL: {e}") + sys.exit(2) +PY +pre_rc="${PIPESTATUS[0]}" +if [ "${pre_rc}" -ne 0 ]; then + echo "[short] FAIL: preflight" | tee -a "${LOG_FILE}" + exit 2 +fi + +#------------------------------------------------------------------------------# +# Launch the training run. +#------------------------------------------------------------------------------# +echo "[short] starting training; this is expected to take 30–60 minutes on Orin AGX" +START_TS="$(date +%s)" +{ + cd "${RUN_DIR}" + # Replay buffer kept at 100k for short run (paper uses 1e6 for full). + # ``replay_start_size`` is bumped above the smoke value so the agent has + # enough random data before learning begins. + all-atari "${ENV_NAME}" "${AGENT_NAME}" \ + --device "${DEVICE}" \ + --frames "${FRAMES}" \ + --logdir "${RUN_DIR}/runs" \ + --hyperparameters \ + replay_buffer_size=100000 \ + replay_start_size=10000 +} 2>&1 | tee -a "${LOG_FILE}" +train_rc="${PIPESTATUS[0]}" +END_TS="$(date +%s)" +echo "[short] training exit code: ${train_rc}; wall-clock: $((END_TS - START_TS))s" | tee -a "${LOG_FILE}" + +if [ "${train_rc}" -ne 0 ]; then + echo "[short] FAIL: training exited ${train_rc}" | tee -a "${LOG_FILE}" + exit 3 +fi + +#------------------------------------------------------------------------------# +# Final metric parse. +#------------------------------------------------------------------------------# +export RUN_DIR +python - <<'PY' 2>&1 | tee -a "${LOG_FILE}" +import json, os, sys +from pathlib import Path + +run_dir = Path(os.environ["RUN_DIR"]) / "runs" +events = sorted(run_dir.rglob("events.out.tfevents.*")) +if not events: + print("[short] WARN: no event files; skipping parse") + sys.exit(0) + +try: + from tensorboard.backend.event_processing.event_accumulator import EventAccumulator +except Exception as e: + print(f"[short] WARN: tensorboard not available ({e}); skipping parse") + sys.exit(0) + +snapshots = [] +for ef in events: + acc = EventAccumulator(str(ef)) + try: + acc.Reload() + except Exception: + continue + for tag in acc.Tags().get("scalars", []): + if tag.endswith("returns/mean") or tag.endswith("returns_100/mean"): + for s in acc.Scalars(tag): + snapshots.append({ + "tag": tag, "step": int(s.step), "value": float(s.value), + }) + +snapshots.sort(key=lambda s: (s["tag"], s["step"])) +out = Path(os.environ["RUN_DIR"]) / "progress.json" +out.write_text(json.dumps(snapshots, indent=2)) +print(f"[short] wrote {out} ({len(snapshots)} samples)") + +if snapshots: + last = snapshots[-1] + print(f"[short] last sample: tag={last['tag']} step={last['step']} value={last['value']:.3f}") +PY + +echo +echo "[short] ✅ PASS — see ${RUN_DIR}" +exit 0 From a9f8f6b28da6cd70d625ad27ef525401c04ff2a5 Mon Sep 17 00:00:00 2001 From: OpenClaw Agent Date: Thu, 16 Apr 2026 23:10:23 -0700 Subject: [PATCH 5/8] feat(reproduce): add full-paper Atari catalog driver run_full_paper.sh is the catalog + driver for every paper experiment that is reproducible on a single Jetson Orin (Atari only). Each catalog entry maps a name to (env, agent, frames, seeds, paper figure, hours estimate, notes). Modes: --list inventory + estimates (no training) --dry-run print exact all-atari invocations, do not execute --only NAME run a single named experiment from the catalog (default) run every in-scope catalog entry serially The DonkeyCar / MUSHR-DL case study from the paper is intentionally listed as out-of-scope and never executed: it requires a separate x86 host running the Unity DonkeySim, so single-Orin reproduction is impossible. The header and reproduce/README.md both link to the manual setup path for users who really need those numbers. Output is written to /experiment/zexin/R3/reproduce/runs/full_/ with per-experiment subdirs and an INDEX.md summary. --- reproduce/run_full_paper.sh | 165 ++++++++++++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100755 reproduce/run_full_paper.sh diff --git a/reproduce/run_full_paper.sh b/reproduce/run_full_paper.sh new file mode 100755 index 0000000..104a665 --- /dev/null +++ b/reproduce/run_full_paper.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash +# reproduce/run_full_paper.sh — driver for the full RTSS 2023 R³ reproduction. +# +# This is the *catalog* + *driver* for every experiment that maps to a figure +# or table in the published paper: +# +# Li et al., "R³: On-device Real-Time Deep Reinforcement Learning for +# Autonomous Robotics," IEEE RTSS 2023. arXiv:2308.15039 +# +# WHAT IT DOES +# ------------ +# * With ``--list`` : prints the catalog (env / agent / frames / +# estimated wall-time / target figure) and exits 0. +# * With ``--dry-run`` : prints the exact ``all-atari`` invocations that +# would run, but does not actually launch them. +# * With no flags : runs every IN-SCOPE experiment serially. +# * With ``--only NAME`` : runs only the named experiment from the catalog. +# +# Out-of-scope reproductions +# -------------------------- +# The DonkeyCar / MUSHR-DL case study (paper §VII) requires a separate x86 PC +# running the Unity ``donkey_sim`` simulator; the Jetson is the *agent*, not +# the simulator. Single-Orin reproduction is therefore IMPOSSIBLE for those +# experiments and they are intentionally excluded here. See ``reproduce/README.md`` +# section "Skipped Reproductions" for the full rationale and the manual setup +# you need if you want to run them yourself. +# +# BUDGET WARNING +# -------------- +# Even the in-scope (Atari only) catalog below sums to multiple GPU-days on +# Orin AGX. The default ``--frames`` per experiment matches the paper's 10M +# frames/run × N seeds per env. Do not launch this casually; prefer ``--list`` +# or ``--dry-run`` first, then pick one experiment with ``--only`` to start. +# +# Output layout (NVMe only — never the eMMC root) +# ----------------------------------------------- +# /experiment/zexin/R3/reproduce/runs/full_/ +# /runs/ ← per-experiment tensorboard subdir +# /run.log ← per-experiment captured output +# INDEX.md ← summary of all attempts +# +set -u +set -o pipefail + +#------------------------------------------------------------------------------# +# Catalog +# ------- +# Each entry is "name|env|agent|frames|seeds|figure|hours_estimate|notes". +# ``hours_estimate`` is rough Orin AGX wall-clock per seed. +# Add new experiments here as the codebase grows; ``--list`` will pick them up. +#------------------------------------------------------------------------------# +CATALOG=( + "atari-pong-dqn|PongNoFrameskip-v4|dqn|10000000|3|Fig.6 / Tab.II|14|Canonical Atari baseline; converges to ~+20" + "atari-breakout-dqn|BreakoutNoFrameskip-v4|dqn|10000000|3|Fig.6 / Tab.II|18|Atari benchmark; ~150 game-score by 10M" + "atari-seaquest-dqn|SeaquestNoFrameskip-v4|dqn|10000000|3|Fig.6 / Tab.II|18|Atari benchmark; sparse reward" + "atari-pong-rainbow|PongNoFrameskip-v4|rainbow|10000000|2|Fig.7 (algo ablation)|16|R³ vs vanilla on Rainbow" + "atari-pong-c51|PongNoFrameskip-v4|c51|10000000|2|Fig.7 (algo ablation)|16|R³ vs vanilla on C51" +) + +OUT_OF_SCOPE=( + "donkeycar-sac : requires Unity DonkeySim on x86 PC; Orin acts only as RL agent. See README#skipped-reproductions." + "donkeycar-ppo : same as above." + "mushr-dl : pre-trained .h5 + DonkeySim required; out of scope for this reproducer." +) + +#------------------------------------------------------------------------------# +# Defaults / knobs. +#------------------------------------------------------------------------------# +DEVICE="${DEVICE:-cuda}" +RUNS_ROOT="${RUNS_ROOT:-/experiment/zexin/R3/reproduce/runs}" +TS="$(date -u +%Y%m%d_%H%M%SZ)" +ROOT_DIR="${RUNS_ROOT}/full_${TS}" +INDEX="${ROOT_DIR}/INDEX.md" + +MODE="run" +ONLY="" + +while [ "$#" -gt 0 ]; do + case "$1" in + --list) MODE="list"; shift ;; + --dry-run) MODE="dry"; shift ;; + --only) ONLY="$2"; shift 2 ;; + -h|--help) + sed -n '1,40p' "$0" + exit 0 + ;; + *) echo "unknown flag: $1" >&2; exit 64 ;; + esac +done + +#------------------------------------------------------------------------------# +# --list mode. +#------------------------------------------------------------------------------# +if [ "${MODE}" = "list" ]; then + echo "# In-scope experiments (Atari, single-Orin reproducible)" + printf "\n%-22s %-28s %-8s %-10s %-5s %-22s %-6s %s\n" \ + "NAME" "ENV" "AGENT" "FRAMES" "SEEDS" "PAPER FIGURE" "HOURS" "NOTES" + printf -- "-%.0s" {1..140}; echo + for entry in "${CATALOG[@]}"; do + IFS='|' read -r name env agent frames seeds fig hours notes <<<"${entry}" + printf "%-22s %-28s %-8s %-10s %-5s %-22s %-6s %s\n" \ + "${name}" "${env}" "${agent}" "${frames}" "${seeds}" "${fig}" "${hours}" "${notes}" + done + echo + echo "# Out-of-scope (require external setup; NOT runnable here)" + for s in "${OUT_OF_SCOPE[@]}"; do + echo " - ${s}" + done + exit 0 +fi + +#------------------------------------------------------------------------------# +# Run / dry-run. +#------------------------------------------------------------------------------# +mkdir -p "${ROOT_DIR}" +{ + echo "# R³ full-paper reproduction batch" + echo + echo "Started: $(date -u --iso-8601=seconds)" + echo "Device : ${DEVICE}" + echo "Mode : ${MODE}${ONLY:+ (only=${ONLY})}" + echo + echo "| Name | Env | Agent | Frames | Seeds | Figure | Status |" + echo "|------|-----|-------|--------|-------|--------|--------|" +} > "${INDEX}" + +run_one() { + local name="$1" env="$2" agent="$3" frames="$4" seeds="$5" fig="$6" + local exp_dir="${ROOT_DIR}/${name}" + mkdir -p "${exp_dir}/runs" + local exp_log="${exp_dir}/run.log" + local status="PENDING" + + for ((seed = 1; seed <= seeds; seed++)); do + local cmd=(all-atari "${env}" "${agent}" + --device "${DEVICE}" + --frames "${frames}" + --logdir "${exp_dir}/runs/seed_${seed}") + echo "[full] ${name} seed=${seed}: ${cmd[*]}" + if [ "${MODE}" = "dry" ]; then + continue + fi + if "${cmd[@]}" 2>&1 | tee -a "${exp_log}"; then + status="OK (last seed=${seed})" + else + status="FAIL (seed=${seed})" + break + fi + done + + [ "${MODE}" = "dry" ] && status="DRY-RUN" + echo "| ${name} | ${env} | ${agent} | ${frames} | ${seeds} | ${fig} | ${status} |" >> "${INDEX}" +} + +for entry in "${CATALOG[@]}"; do + IFS='|' read -r name env agent frames seeds fig hours notes <<<"${entry}" + if [ -n "${ONLY}" ] && [ "${ONLY}" != "${name}" ]; then + continue + fi + run_one "${name}" "${env}" "${agent}" "${frames}" "${seeds}" "${fig}" +done + +echo +echo "[full] index -> ${INDEX}" +exit 0 From 581d18e13f5d040ce962584e8e969ab62118a250 Mon Sep 17 00:00:00 2001 From: OpenClaw Agent Date: Thu, 16 Apr 2026 23:10:33 -0700 Subject: [PATCH 6/8] docs(reproduce): add full README index for paper reproduction guide MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit reproduce/README.md is the single entry point for reproducing the RTSS 2023 R^3 paper on the modernized PyTorch 2.8 + Gymnasium + JetPack 6.2 stack. Sections: 1. System requirements (JetPack 6.2 / Orin AGX recommended). 2. Output convention — every script writes under /experiment/zexin/R3/reproduce/runs/ on the NVMe, never the eMMC root. 3. The four scripts (smoke, classic smoke, short, full) with usage, tmux/tensorboard recipes, and exit-code semantics. 4. Skipped reproductions — explicitly documents why DonkeyCar / MUSHR-DL are out of scope on a single Orin (require x86 + Unity DonkeySim). 5. Current status of R^3 runtime modules vs. agent presets, so a reader knows exactly what the smoke / short / full runs prove today. 6. Troubleshooting table covering the most common failure modes (missing ROMs, OOM, disk fills up, command not found, SSH drop). --- reproduce/README.md | 206 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 reproduce/README.md diff --git a/reproduce/README.md b/reproduce/README.md new file mode 100644 index 0000000..e5cc68a --- /dev/null +++ b/reproduce/README.md @@ -0,0 +1,206 @@ +# R³ Paper Reproduction Guide + +This folder is the **single entry point** for reproducing the experiments from: + +> Zexin Li, Aritra Samanta, Yufei Li, Andrea Soltoggio, Hyoseung Kim, Cong Liu. +> **R³: On-Device Real-Time Deep Reinforcement Learning for Autonomous Robotics.** +> *IEEE Real-Time Systems Symposium (RTSS) 2023*, pp. 131–144. +> [arXiv:2308.15039](https://arxiv.org/abs/2308.15039) · [DOI 10.1109/RTSS59052.2023.00021](https://doi.org/10.1109/RTSS59052.2023.00021) + +If you only want to know whether the modernized stack works, run +`bash reproduce/run_smoke.sh`. Everything else here is a graduated path from +"5-minute smoke test" to "full multi-day paper reproduction". + +--- + +## 1. System requirements + +| Component | Tested / recommended | +|------------------|------------------------------------------------------------------| +| Hardware | **NVIDIA Jetson AGX Orin** (also runs on x86 + CUDA GPU) | +| OS | Ubuntu 22.04 (JetPack 6.2) | +| JetPack | **6.2** (CUDA 12.6, cuDNN 9.x) | +| Python | 3.10 (ships with JetPack 6.2) | +| PyTorch | 2.8.x (Jetson AI Lab wheels — see main `README.md` quick-install)| +| Gymnasium | ≥ 0.29 | +| ALE / Atari ROMs | `ale-py` ≥ 0.10 + ROMs registered via `AutoROM --accept-license` | +| Disk | All artifacts go on `/experiment` (NVMe). The eMMC root has < 10 GB free — never write logs/checkpoints to `~`. | + +The complete install recipe lives in the project root [`README.md`](../README.md). +The reproduction scripts assume the venv at `/experiment/zexin/venvs/r3/` +already exists and has the modernized stack installed. + +### One-liner environment activation (Jetson Orin) + +```bash +source /experiment/zexin/venvs/r3/bin/activate +cd /experiment/zexin/R3 +``` + +### Atari ROMs (one-time, only if the smoke preflight fails) + +```bash +pip install 'ale-py' 'autorom[accept-rom-license]' +AutoROM --accept-license +``` + +--- + +## 2. Output convention + +**Every** script in this folder writes under a single tree on the NVMe SSD: + +``` +/experiment/zexin/R3/reproduce/runs/ +├── smoke_/ ← run_smoke.sh +├── short_/ ← run_short.sh +├── full_/ ← run_full_paper.sh +└── short_LATEST -> ... ← convenience symlink +``` + +Override the root with `RUNS_ROOT=/some/other/path` if you really must, but +**never** point it at `~/` or `/home/...` on the Orin — the eMMC root will +fill up and brick the box. See [`AGENTS.md`](../AGENTS.md) for the full disk +layout rule. + +--- + +## 3. The four scripts + +| Script | Wall-clock | Purpose | +|------------------------------|--------------------|--------------------------------------------------------------------------| +| `run_smoke.sh` | 5–10 min | End-to-end pipeline gate on **Atari Pong** (≈20k frames). Exit 0 = green.| +| `run_smoke_classic.sh` | 3–5 min | Same idea but on CartPole-v1; useful when you don't have Atari ROMs yet. | +| `run_short.sh` | 30–60 min | "Does R³ start learning?" — DQN/Pong, 250k frames, daemonizable. | +| `run_full_paper.sh` | days | Catalog + driver for all in-scope paper experiments. Use `--list` first. | +| `list_experiments.py` | seconds | Inventory of every experiment script and preset under ALL. | + +### 3.1 Smoke test — `run_smoke.sh` + +```bash +bash reproduce/run_smoke.sh +# or +FRAMES=10000 bash reproduce/run_smoke.sh +ENV=BreakoutNoFrameskip-v4 bash reproduce/run_smoke.sh +``` + +What it checks, in order: + +1. R³ algorithmic unit tests (`autonomous-learning-library/all/r3/*_test.py`). +2. ALE / ROM availability (Pong is constructible). +3. A real `all-atari` training run for `FRAMES` frames. +4. Tensorboard log parse: prints final `returns/mean`. + +Exit codes: `0` PASS, `1` unit-test fail, `2` ALE/ROM missing, `3` training +crash, `4` log-parse failure. + +### 3.2 Short paper validation — `run_short.sh` + +```bash +# Recommended: detach via tmux so SSH disconnects don't kill the run. +tmux new-session -d -s r3-short \ + "source /experiment/zexin/venvs/r3/bin/activate && \ + cd /experiment/zexin/R3 && \ + bash reproduce/run_short.sh" + +# Inspect progress without attaching to the tmux pane: +tail -F /experiment/zexin/R3/reproduce/runs/short_LATEST/short.log + +# Attach to the tmux pane: +tmux attach -t r3-short +# (detach with Ctrl-b d) + +# Tensorboard (from any host that can reach the Orin on port 6006): +tensorboard --logdir /experiment/zexin/R3/reproduce/runs/short_LATEST/runs --bind_all +``` + +The script also accepts `--background`, which `setsid+nohup`-daemonizes itself +without needing `tmux`. Use that if `tmux` is not available. + +### 3.3 Full paper reproduction — `run_full_paper.sh` + +This is a *catalog* and a *driver*. It is **not** intended to be run blindly — +the full catalog is multiple GPU-days. Always start with `--list`. + +```bash +bash reproduce/run_full_paper.sh --list # inventory + estimates +bash reproduce/run_full_paper.sh --dry-run # print exact commands, do not run +bash reproduce/run_full_paper.sh --only atari-pong-dqn # one experiment, all seeds +``` + +Each experiment in the catalog maps to a figure or table in the RTSS 2023 +paper. The current (Atari-only) catalog: + +| Name | Env | Agent | Frames | Seeds | Paper figure | +|-----------------------|---------------------------|---------|------------|-------|---------------------------| +| `atari-pong-dqn` | `PongNoFrameskip-v4` | dqn | 10 000 000 | 3 | Fig. 6 / Table II | +| `atari-breakout-dqn` | `BreakoutNoFrameskip-v4` | dqn | 10 000 000 | 3 | Fig. 6 / Table II | +| `atari-seaquest-dqn` | `SeaquestNoFrameskip-v4` | dqn | 10 000 000 | 3 | Fig. 6 / Table II | +| `atari-pong-rainbow` | `PongNoFrameskip-v4` | rainbow | 10 000 000 | 2 | Fig. 7 (algorithm ablation)| +| `atari-pong-c51` | `PongNoFrameskip-v4` | c51 | 10 000 000 | 2 | Fig. 7 (algorithm ablation)| + +### 3.4 Inventory — `list_experiments.py` + +```bash +python reproduce/list_experiments.py # markdown +python reproduce/list_experiments.py --json # machine-readable +``` + +Scans `autonomous-learning-library/{examples,all/scripts,all/r3,all/presets}` +and emits a per-category table with paths, default frame counts, env hints, +and agent hints. + +--- + +## 4. Skipped reproductions + +The original RTSS 2023 paper also contains an autonomous-driving case study +built on the [`gym-donkeycar`](https://github.com/tawnkramer/gym-donkeycar) +simulator. Those experiments are **out of scope** for this on-Orin reproducer +because they require additional hosts and assets that the Jetson alone cannot +provide: + +- **DonkeyCar / DonkeySim experiments** require a separate x86 / Ubuntu PC + running the Unity-based `donkey_sim` binary. The Jetson is the RL agent only; + the simulator process is heavy and Linux-x86-only at the version pinned by + the paper. +- **MUSHR-DL** depends on pre-trained `.h5` models *and* the same DonkeySim + binary. There is no single-Orin path. + +If you specifically need to reproduce those numbers, follow the legacy install +section at the bottom of the project [`README.md`](../README.md), set up the +PC + simulator, and use the scripts under `MUSHR-DL/`. They are intentionally +omitted from `run_full_paper.sh`. + +--- + +## 5. R³ runtime modules vs. agent presets — current status + +The R³ algorithmic core (memory budget, deadline, batch-size control, replay +buffer resize) lives in +[`autonomous-learning-library/all/r3/`](../autonomous-learning-library/all/r3/). +It is unit-tested but on the modernized branch it is **not yet wired** as a +default hook into the upstream agent presets (DQN, Rainbow, etc.). The +reproduction scripts therefore exercise: + +- **Vanilla ALL** training to confirm the modernized stack converges normally, + and +- The R³ unit tests to confirm the algorithmic equations match the paper. + +Once the wiring PRs land on `main`, this guide will be updated to flip the +appropriate `--hyperparameters r3=true` flag (or equivalent) on each +reproduction. Until then, the catalog above represents the upper-bound DQN +baselines; R³'s memory/time savings are demonstrated by the unit suite. + +--- + +## 6. Troubleshooting + +| Symptom | Likely cause / fix | +|-------------------------------------------------|--------------------------------------------------------------------------------------| +| `cannot import ale_py` | `pip install 'ale-py' 'autorom[accept-rom-license]'` then `AutoROM --accept-license`| +| `Environment PongNoFrameskip-v4 doesn't exist` | ROMs not installed; same fix as above. | +| Out-of-memory on Orin | Lower `replay_buffer_size` via `--hyperparameters replay_buffer_size=50000`. | +| Disk fills up | You wrote to `~/` instead of `/experiment/`. Move data to `/experiment/zexin/...`. | +| Run died on SSH disconnect | Use `tmux` (recommended) or `bash reproduce/run_short.sh --background`. | +| `all-atari: command not found` | The venv is not activated; `source /experiment/zexin/venvs/r3/bin/activate`. | From 462a39b3066a2d82392bfc8569e32091d970c85d Mon Sep 17 00:00:00 2001 From: OpenClaw Agent Date: Thu, 16 Apr 2026 23:12:44 -0700 Subject: [PATCH 7/8] fix(reproduce): pass short Atari env names to all-atari CLI ALL's all-atari CLI (autonomous-learning-library/all/scripts/train_atari.py + all/environments/atari.py) automatically appends 'NoFrameskip-v4' to the env name passed on the command line. Passing the full 'PongNoFrameskip-v4' therefore produced 'PongNoFrameskip-v4NoFrameskip' which gymnasium does not know about. Use the short names ('Pong', 'Breakout', 'Seaquest') in: - reproduce/run_smoke.sh - reproduce/run_short.sh - reproduce/run_full_paper.sh catalog - reproduce/README.md catalog table + ENV override example Discovered while running the smoke test on Jetson Orin AGX. --- reproduce/README.md | 12 ++++++------ reproduce/run_full_paper.sh | 12 +++++++----- reproduce/run_short.sh | 4 +++- reproduce/run_smoke.sh | 7 +++++-- 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/reproduce/README.md b/reproduce/README.md index e5cc68a..36585c8 100644 --- a/reproduce/README.md +++ b/reproduce/README.md @@ -81,7 +81,7 @@ layout rule. bash reproduce/run_smoke.sh # or FRAMES=10000 bash reproduce/run_smoke.sh -ENV=BreakoutNoFrameskip-v4 bash reproduce/run_smoke.sh +ENV=Breakout bash reproduce/run_smoke.sh # short name; ALL appends NoFrameskip-v4 ``` What it checks, in order: @@ -133,11 +133,11 @@ paper. The current (Atari-only) catalog: | Name | Env | Agent | Frames | Seeds | Paper figure | |-----------------------|---------------------------|---------|------------|-------|---------------------------| -| `atari-pong-dqn` | `PongNoFrameskip-v4` | dqn | 10 000 000 | 3 | Fig. 6 / Table II | -| `atari-breakout-dqn` | `BreakoutNoFrameskip-v4` | dqn | 10 000 000 | 3 | Fig. 6 / Table II | -| `atari-seaquest-dqn` | `SeaquestNoFrameskip-v4` | dqn | 10 000 000 | 3 | Fig. 6 / Table II | -| `atari-pong-rainbow` | `PongNoFrameskip-v4` | rainbow | 10 000 000 | 2 | Fig. 7 (algorithm ablation)| -| `atari-pong-c51` | `PongNoFrameskip-v4` | c51 | 10 000 000 | 2 | Fig. 7 (algorithm ablation)| +| `atari-pong-dqn` | `Pong` (→ `PongNoFrameskip-v4`) | dqn | 10 000 000 | 3 | Fig. 6 / Table II | +| `atari-breakout-dqn` | `Breakout` (→ `BreakoutNoFrameskip-v4`) | dqn | 10 000 000 | 3 | Fig. 6 / Table II | +| `atari-seaquest-dqn` | `Seaquest` (→ `SeaquestNoFrameskip-v4`) | dqn | 10 000 000 | 3 | Fig. 6 / Table II | +| `atari-pong-rainbow` | `Pong` | rainbow | 10 000 000 | 2 | Fig. 7 (algorithm ablation)| +| `atari-pong-c51` | `Pong` | c51 | 10 000 000 | 2 | Fig. 7 (algorithm ablation)| ### 3.4 Inventory — `list_experiments.py` diff --git a/reproduce/run_full_paper.sh b/reproduce/run_full_paper.sh index 104a665..f348b32 100755 --- a/reproduce/run_full_paper.sh +++ b/reproduce/run_full_paper.sh @@ -49,12 +49,14 @@ set -o pipefail # ``hours_estimate`` is rough Orin AGX wall-clock per seed. # Add new experiments here as the codebase grows; ``--list`` will pick them up. #------------------------------------------------------------------------------# +# NOTE: ALL's all-atari CLI appends "NoFrameskip-v4" itself, so pass the +# short env name ("Pong", "Breakout", "Seaquest") here. CATALOG=( - "atari-pong-dqn|PongNoFrameskip-v4|dqn|10000000|3|Fig.6 / Tab.II|14|Canonical Atari baseline; converges to ~+20" - "atari-breakout-dqn|BreakoutNoFrameskip-v4|dqn|10000000|3|Fig.6 / Tab.II|18|Atari benchmark; ~150 game-score by 10M" - "atari-seaquest-dqn|SeaquestNoFrameskip-v4|dqn|10000000|3|Fig.6 / Tab.II|18|Atari benchmark; sparse reward" - "atari-pong-rainbow|PongNoFrameskip-v4|rainbow|10000000|2|Fig.7 (algo ablation)|16|R³ vs vanilla on Rainbow" - "atari-pong-c51|PongNoFrameskip-v4|c51|10000000|2|Fig.7 (algo ablation)|16|R³ vs vanilla on C51" + "atari-pong-dqn|Pong|dqn|10000000|3|Fig.6 / Tab.II|14|Canonical Atari baseline; converges to ~+20" + "atari-breakout-dqn|Breakout|dqn|10000000|3|Fig.6 / Tab.II|18|Atari benchmark; ~150 game-score by 10M" + "atari-seaquest-dqn|Seaquest|dqn|10000000|3|Fig.6 / Tab.II|18|Atari benchmark; sparse reward" + "atari-pong-rainbow|Pong|rainbow|10000000|2|Fig.7 (algo ablation)|16|R³ vs vanilla on Rainbow" + "atari-pong-c51|Pong|c51|10000000|2|Fig.7 (algo ablation)|16|R³ vs vanilla on C51" ) OUT_OF_SCOPE=( diff --git a/reproduce/run_short.sh b/reproduce/run_short.sh index a0c2e33..a1c6ce4 100755 --- a/reproduce/run_short.sh +++ b/reproduce/run_short.sh @@ -43,7 +43,9 @@ set -o pipefail # Knobs. #------------------------------------------------------------------------------# FRAMES="${FRAMES:-250000}" -ENV_NAME="${ENV:-PongNoFrameskip-v4}" +# NOTE: ALL's all-atari CLI wraps the env name with "NoFrameskip-v4" itself, +# so pass the *short* name (e.g. "Pong", not "PongNoFrameskip-v4"). +ENV_NAME="${ENV:-Pong}" AGENT_NAME="${AGENT:-dqn}" DEVICE="${DEVICE:-cuda}" RUNS_ROOT="${RUNS_ROOT:-/experiment/zexin/R3/reproduce/runs}" diff --git a/reproduce/run_smoke.sh b/reproduce/run_smoke.sh index e4fe62f..e890d5c 100755 --- a/reproduce/run_smoke.sh +++ b/reproduce/run_smoke.sh @@ -55,7 +55,10 @@ set -o pipefail # Knobs (override via env vars). #------------------------------------------------------------------------------# FRAMES="${FRAMES:-20000}" -ENV_NAME="${ENV:-PongNoFrameskip-v4}" +# NOTE: ALL's all-atari CLI wraps the env name with "NoFrameskip-v4" itself +# (see autonomous-learning-library/all/environments/atari.py). So pass the +# *short* name here (e.g. "Pong", not "PongNoFrameskip-v4"). +ENV_NAME="${ENV:-Pong}" AGENT_NAME="${AGENT:-dqn}" DEVICE="${DEVICE:-cuda}" RUNS_ROOT="${RUNS_ROOT:-/experiment/zexin/R3/reproduce/runs}" @@ -127,7 +130,7 @@ fi # Step 3 — micro Atari training run via all-atari. #------------------------------------------------------------------------------# echo -echo "[smoke] step 3/4 — micro Atari training: ${AGENT_NAME} on ${ENV_NAME} for ${FRAMES} frames" +echo "[smoke] step 3/4 — micro Atari training: ${AGENT_NAME} on ${ENV_NAME}NoFrameskip-v4 for ${FRAMES} frames" { cd "${RUN_DIR}" # Tiny replay buffer to keep Orin RAM happy at smoke scale; full paper From b9ee861d594f2eb4f1a1a3b1be55f72f527588be Mon Sep 17 00:00:00 2001 From: OpenClaw Agent Date: Thu, 16 Apr 2026 23:13:22 -0700 Subject: [PATCH 8/8] fix(atari): register ale_py envs explicitly for ale-py >= 0.11 Starting in ale-py 0.11, the package no longer auto-registers Atari envs into gymnasium on import. Calling gymnasium.make('PongNoFrameskip-v4') therefore raises NameNotFound even when ale_py is installed and Pong's ROM is bundled, which broke the reproduce/run_smoke.sh Atari path on Jetson Orin (JetPack 6.2 venv ships ale-py 0.11.2). Add a guarded gymnasium.register_envs(ale_py) at the top of all/environments/atari.py. The try/except keeps the import optional so users who do not want Atari support do not need ale-py installed. Tested on Orin AGX with the smoke run. --- autonomous-learning-library/all/environments/atari.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/autonomous-learning-library/all/environments/atari.py b/autonomous-learning-library/all/environments/atari.py index c889c94..10ee281 100644 --- a/autonomous-learning-library/all/environments/atari.py +++ b/autonomous-learning-library/all/environments/atari.py @@ -1,6 +1,16 @@ import gymnasium import torch +# ale-py >= 0.11 stopped auto-registering Atari envs into gymnasium. Without +# this explicit register call, ``gymnasium.make("PongNoFrameskip-v4")`` fails +# with NameNotFound even when ale_py is importable. Guarded so the import is +# optional for users who don't want Atari at all. +try: + import ale_py as _ale_py + gymnasium.register_envs(_ale_py) +except Exception: + pass + from all.core import State from ._environment import Environment