From 38499104ed91df334aa5bac1c8e55fb02776eadc Mon Sep 17 00:00:00 2001 From: zhiheng-yang Date: Fri, 4 Sep 2026 00:54:05 +0200 Subject: [PATCH] fix(rollout): clear stale LLM trajectory on rollout reuse --- src/benchflow/rollout/_setup.py | 1 + tests/test_sdk_internals.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/src/benchflow/rollout/_setup.py b/src/benchflow/rollout/_setup.py index 33c95c7e8..24762606b 100644 --- a/src/benchflow/rollout/_setup.py +++ b/src/benchflow/rollout/_setup.py @@ -273,6 +273,7 @@ def _init_rollout( rollout_dir.mkdir(parents=True, exist_ok=True) for subdir in ("agent", "verifier", "artifacts", "trajectory"): (rollout_dir / subdir).mkdir(exist_ok=True) + (rollout_dir / "trajectory" / "llm_trajectory.jsonl").unlink(missing_ok=True) return task, rollout_dir, rollout_paths, started_at, job_name, rollout_name diff --git a/tests/test_sdk_internals.py b/tests/test_sdk_internals.py index df571c90c..a60090683 100644 --- a/tests/test_sdk_internals.py +++ b/tests/test_sdk_internals.py @@ -382,6 +382,27 @@ def test_custom_rollout_name(self, task_dir, tmp_path): ) assert rollout_name == "custom-trial" + def test_reused_rollout_drops_previous_llm_trajectory(self, task_dir, tmp_path): + """Drops a prior provider trajectory when a reused rollout has no new one.""" + jobs_dir = tmp_path / "jobs" + _, rollout_dir, _, _, _, _ = self._init( + task_dir, + job_name="job", + rollout_name="rollout", + jobs_dir=jobs_dir, + ) + trajectory = rollout_dir / "trajectory" / "llm_trajectory.jsonl" + trajectory.write_text('{"completion":"stale"}\n') + + self._init( + task_dir, + job_name="job", + rollout_name="rollout", + jobs_dir=jobs_dir, + ) + + assert not trajectory.exists() + def test_started_at_is_datetime(self, task_dir, tmp_path): _, _, _, started_at, _, _ = self._init(task_dir, jobs_dir=tmp_path / "jobs") assert isinstance(started_at, datetime)