diff --git a/.github/workflows/internal-preview-release.yml b/.github/workflows/internal-preview-release.yml index 48cd15174..4e55ba55e 100644 --- a/.github/workflows/internal-preview-release.yml +++ b/.github/workflows/internal-preview-release.yml @@ -105,7 +105,8 @@ jobs: - name: Compute internal preview version id: version run: >- - uv run --with packaging --no-project python tools/release_version.py + uv run --with packaging --with pyyaml --no-project python + tools/release_version.py internal-preview --run-number "${{ steps.provenance.outputs.source_run_number }}" diff --git a/.github/workflows/public-release.yml b/.github/workflows/public-release.yml index 11ead97d8..671aa3cdc 100644 --- a/.github/workflows/public-release.yml +++ b/.github/workflows/public-release.yml @@ -44,11 +44,16 @@ jobs: exit 1 fi + # Runs before the build so a stale CITATION.cff stops the release while it + # is still reversible; PyPI publishes are not. PyYAML is explicit because + # `--no-project` skips the project's own dependencies. - name: Validate release tag id: version env: TAG_NAME: ${{ github.ref_name }} - run: uv run --with packaging --no-project python tools/release_version.py public-release + run: >- + uv run --with packaging --with pyyaml --no-project + python tools/release_version.py public-release - name: Build distributions run: uv build --no-sources diff --git a/.zenodo.json b/.zenodo.json new file mode 100644 index 000000000..4ecf5fc08 --- /dev/null +++ b/.zenodo.json @@ -0,0 +1,28 @@ +{ + "upload_type": "software", + "title": "BenchFlow: framework for RL environments for LLM agents", + "description": "BenchFlow is a framework for building RL environments to evaluate and train LLM agents. Built on the Agent Client Protocol (ACP), it provides Scene-based multi-turn, multi-agent, and multi-model evaluation in shared sandboxes — without Docker Compose or sidecar containers. Supported use cases include interactive user simulation, code-review loops, bring-your-own-skill (BYOS) skill generation, multi-turn iterative refinement, cross-model review (cheap coder + strong reviewer), and stateful service tasks against live mock APIs (Gmail, Calendar, Docs, Drive, Slack). See docs/use-cases.md.", + "creators": [ + { + "name": "TODO BEFORE MERGE - replace with the agreed author list (\"Family, Given\" per entry, plus orcid and affiliation where known). Zenodo mints a permanent DOI from this file, so a placeholder that reaches main becomes permanent authorship." + } + ], + "license": "Apache-2.0", + "access_right": "open", + "keywords": [ + "benchmark", + "llm-agents", + "acp", + "agent-evaluation", + "multi-turn", + "terminal-bench", + "skillsbench" + ], + "related_identifiers": [ + { + "relation": "isVariantFormOf", + "identifier": "https://pypi.org/project/benchflow/", + "resource_type": "software" + } + ] +} diff --git a/CITATION.cff b/CITATION.cff index 1cf1a26a8..b55c1e3c3 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -9,8 +9,8 @@ authors: repository-code: "https://github.com/benchflow-ai/benchflow" url: "https://github.com/benchflow-ai/benchflow" license: Apache-2.0 -version: 0.6.9 -date-released: 2026-08-15 +version: 0.7.4 +date-released: 2026-08-17 keywords: - benchmark - llm-agents diff --git a/docs/release.md b/docs/release.md index f4505b1b9..402874b85 100644 --- a/docs/release.md +++ b/docs/release.md @@ -127,13 +127,17 @@ runs `uv publish`. Public release: 1. Update `pyproject.toml` from the next `.dev0` version to the final public - version. + version, and set `CITATION.cff` `version` and `date-released` to that same + release. 2. Merge the release PR to `main`. 3. Push a matching release tag. -4. `.github/workflows/public-release.yml` validates the tag, publishes to PyPI, - and creates a GitHub Release. The workflow refuses tags whose commits are - not contained in `origin/main`. -5. Bump `main` to the next `.dev0`. +4. `.github/workflows/public-release.yml` validates the tag against + `pyproject.toml` and `CITATION.cff`, publishes to PyPI, and creates a GitHub + Release. The workflow refuses tags whose commits are not contained in + `origin/main`, and tags whose `CITATION.cff` still names an older release. +5. Bump `main` to the next `.dev0`. Leave `CITATION.cff` on the version just + released: it names the last published release, not the line under + development. ## One-Time PyPI Setup @@ -154,3 +158,40 @@ Create matching GitHub environments: The workflows build with `uv build --no-sources`, check distributions with `twine check`, and publish with `uv publish`. + +## Zenodo Archiving + +Each published GitHub Release is archived on Zenodo through the official +GitHub integration, which snapshots the tag's source tree and mints a version +DOI for it. Zenodo also maintains a concept DOI that always resolves to the +newest archived version; cite the concept DOI, not a version DOI, so the +citation never goes stale. + +`.zenodo.json` at the repository root supplies the archive metadata. Without it +Zenodo derives authorship from GitHub contributor statistics, which is not the +project's author list. Its `version` and `publication_date` are deliberately +absent: Zenodo takes both from the tag and the GitHub Release, so pinning them +would go stale on every release. + +`CITATION.cff` is the human-facing citation record and names the **last +published** release. On `main` it therefore sits one release behind the `.devN` +version in `pyproject.toml`, which is correct, not drift. The tag-driven +`public-release.yml` validation is what keeps it honest: a tag whose +`CITATION.cff` names a different version fails before anything is built or +published. + +One-time setup, which requires **admin** rights on the repository: + +1. Sign in to with the GitHub account, and grant Zenodo the + GitHub authorization it asks for. +2. Open Zenodo's GitHub page () and + flip the switch for `benchflow-ai/benchflow` on. Zenodo installs a release + webhook; it archives releases published *after* the switch is flipped, so the + first archived version is the next release, not the current one. +3. After that release, confirm on Zenodo that the record's authors, title, and + license came from `.zenodo.json` rather than from contributor statistics. +4. Add the concept DOI to `README.md` and to `CITATION.cff` once it exists. + +Flip the switch only after a `.zenodo.json` carrying the real author list has +merged to `main`. Zenodo archives the tagged commit, and a published DOI cannot +be withdrawn -- an incomplete author list becomes permanent public metadata. diff --git a/tests/test_citation_metadata.py b/tests/test_citation_metadata.py new file mode 100644 index 000000000..5db924f74 --- /dev/null +++ b/tests/test_citation_metadata.py @@ -0,0 +1,96 @@ +"""Parity coverage for the citation and Zenodo archiving metadata files.""" + +from __future__ import annotations + +import json +import tomllib +from pathlib import Path + +import pytest +import yaml +from packaging.version import Version + +REPO_ROOT = Path(__file__).resolve().parent.parent + +# Zenodo derives a release's version and publication date from the git tag and +# the GitHub Release, so pinning either here would go stale on every release -- +# the exact drift the tag-time CITATION.cff gate exists to prevent. +ZENODO_DERIVED_KEYS = ("version", "publication_date") + +CREATORS_PLACEHOLDER = "TODO BEFORE MERGE" + + +@pytest.fixture(scope="module") +def pyproject() -> dict: + return tomllib.loads((REPO_ROOT / "pyproject.toml").read_text(encoding="utf-8")) + + +@pytest.fixture(scope="module") +def citation() -> dict: + return yaml.safe_load((REPO_ROOT / "CITATION.cff").read_text(encoding="utf-8")) + + +@pytest.fixture(scope="module") +def zenodo() -> dict: + return json.loads((REPO_ROOT / ".zenodo.json").read_text(encoding="utf-8")) + + +def test_zenodo_declares_software_upload(zenodo: dict) -> None: + """Guards the Zenodo archiving deposit shape.""" + assert zenodo["upload_type"] == "software" + assert zenodo["access_right"] == "open" + + +def test_zenodo_matches_citation_prose(citation: dict, zenodo: dict) -> None: + """Guards the Zenodo archiving metadata against drifting from CITATION.cff.""" + assert zenodo["title"] == citation["title"] + assert zenodo["description"] == citation["abstract"] + assert zenodo["keywords"] == citation["keywords"] + + +def test_license_is_declared_identically_everywhere( + pyproject: dict, citation: dict, zenodo: dict +) -> None: + """Guards the Zenodo archiving license claim against a three-way mismatch.""" + assert pyproject["project"]["license"]["text"] == "Apache-2.0" + assert citation["license"] == "Apache-2.0" + assert zenodo["license"] == "Apache-2.0" + + +def test_zenodo_omits_release_derived_fields(zenodo: dict) -> None: + """Guards the Zenodo archiving metadata against hardcoding a release version.""" + for key in ZENODO_DERIVED_KEYS: + assert key not in zenodo + + +def test_citation_never_names_an_unpublished_version( + pyproject: dict, citation: dict +) -> None: + """Guards CITATION.cff against naming a version that was never released. + + CITATION.cff names the last published release, so on `main` it is either + equal to the staged public version or behind the next `.devN` line. This + catches a citation file that has run ahead of the project; a citation file + that lags several releases behind is caught at tag time instead, by + `tools/release_version.py public-release`. + """ + project_release = Version(pyproject["project"]["version"]).base_version + + assert Version(str(citation["version"])) <= Version(project_release) + + +def test_zenodo_creators_are_resolved(zenodo: dict) -> None: + """Blocks merging the Zenodo archiving PR with placeholder authorship. + + Zenodo mints a permanent DOI from `.zenodo.json`, and a published DOI cannot + be withdrawn. This test is expected to fail until the agreed author list -- + names, ORCIDs, affiliations -- replaces the placeholder. + """ + creators = zenodo["creators"] + + assert creators, ".zenodo.json must declare at least one creator." + for creator in creators: + assert CREATORS_PLACEHOLDER not in creator["name"], ( + "Replace the .zenodo.json creators placeholder with the agreed " + "author list before merging; Zenodo authorship is permanent." + ) diff --git a/tests/test_release_version.py b/tests/test_release_version.py index 2f4e910f0..bdc15fe26 100644 --- a/tests/test_release_version.py +++ b/tests/test_release_version.py @@ -1,17 +1,42 @@ from __future__ import annotations +from datetime import date from pathlib import Path import pytest from tools.release_version import ( + CitationRelease, InternalPreviewDecision, ReleaseVersionError, compute_internal_preview_version, main, + read_citation_release, + validate_public_release_citation, validate_public_release_version, ) +CITATION_TEMPLATE = """cff-version: 1.2.0 +message: "If you use benchflow in your research, please cite it as below." +title: "BenchFlow" +type: software +authors: + - name: "BenchFlow team" +license: Apache-2.0 +version: {version} +date-released: {date_released} +""" + + +def _citation( + tmp_path: Path, *, version: str = "0.5.1", date_released: str = "2026-01-02" +) -> Path: + citation = tmp_path / "CITATION.cff" + citation.write_text( + CITATION_TEMPLATE.format(version=version, date_released=date_released) + ) + return citation + @pytest.mark.parametrize( ("version", "run_number", "expected"), @@ -122,6 +147,8 @@ def test_public_release_cli_writes_github_outputs( "public-release", "--pyproject", str(pyproject), + "--citation", + str(_citation(tmp_path)), "--tag", "v0.5.1", ] @@ -130,3 +157,171 @@ def test_public_release_cli_writes_github_outputs( ) assert output.read_text() == "version=0.5.1\n" + + +def test_citation_release_reads_unquoted_scalars(tmp_path: Path) -> None: + """Guards the Zenodo archiving citation gate against YAML scalar surprises.""" + citation = _citation(tmp_path, version="0.5.1", date_released="2026-01-02") + + assert read_citation_release(citation) == CitationRelease( + version="0.5.1", date_released=date(2026, 1, 2) + ) + + +def test_citation_release_reads_quoted_scalars(tmp_path: Path) -> None: + """Guards the Zenodo archiving citation gate for quoted CITATION.cff values.""" + citation = _citation(tmp_path, version='"1.0"', date_released='"2026-01-02"') + + assert read_citation_release(citation) == CitationRelease( + version="1.0", date_released=date(2026, 1, 2) + ) + + +def test_citation_release_reads_float_shaped_version(tmp_path: Path) -> None: + """Guards the Zenodo archiving citation gate for a `1.0` release line. + + YAML parses an unquoted `version: 1.0` as a float, so the reader must accept + that shape instead of rejecting a legitimate release. + """ + citation = _citation(tmp_path, version="1.0") + + assert read_citation_release(citation).version == "1.0" + + +def test_citation_release_rejects_missing_file(tmp_path: Path) -> None: + """Guards the Zenodo archiving citation gate against a deleted citation file.""" + with pytest.raises(ReleaseVersionError, match="is missing"): + read_citation_release(tmp_path / "CITATION.cff") + + +def test_citation_release_rejects_malformed_yaml(tmp_path: Path) -> None: + """Guards the Zenodo archiving citation gate against unparsable YAML.""" + citation = tmp_path / "CITATION.cff" + citation.write_text("version: [0.5.1\n") + + with pytest.raises(ReleaseVersionError, match="as YAML"): + read_citation_release(citation) + + +def test_citation_release_rejects_non_mapping(tmp_path: Path) -> None: + """Guards the Zenodo archiving citation gate against a non-mapping document.""" + citation = tmp_path / "CITATION.cff" + citation.write_text("- 0.5.1\n") + + with pytest.raises(ReleaseVersionError, match="YAML mapping"): + read_citation_release(citation) + + +@pytest.mark.parametrize( + ("body", "match"), + [ + ("date-released: 2026-01-02\n", "`version`"), + ("version: 0.5.1\n", "`date-released`"), + ("version: 0.5.1\ndate-released: not-a-date\n", "unparsable"), + ], +) +def test_citation_release_rejects_incomplete_metadata( + tmp_path: Path, body: str, match: str +) -> None: + """Guards the Zenodo archiving citation gate against incomplete metadata.""" + citation = tmp_path / "CITATION.cff" + citation.write_text(body) + + with pytest.raises(ReleaseVersionError, match=match): + read_citation_release(citation) + + +def test_public_release_citation_accepts_matching_release() -> None: + """Guards the Zenodo archiving citation gate happy path.""" + citation = CitationRelease(version="0.5.1", date_released=date(2026, 1, 2)) + + validate_public_release_citation("0.5.1", citation, date(2026, 1, 2)) + + +def test_public_release_citation_accepts_pep440_equivalent_version() -> None: + """Guards the Zenodo archiving citation gate against padding-only mismatches.""" + citation = CitationRelease(version="0.5.1.0", date_released=date(2026, 1, 2)) + + validate_public_release_citation("0.5.1", citation, date(2026, 1, 2)) + + +def test_public_release_citation_rejects_stale_version() -> None: + """Guards the Zenodo archiving citation gate against archiving a stale version.""" + citation = CitationRelease(version="0.5.0", date_released=date(2026, 1, 2)) + + with pytest.raises(ReleaseVersionError, match=r"CITATION\.cff records version"): + validate_public_release_citation("0.5.1", citation, date(2026, 1, 2)) + + +def test_public_release_citation_tolerates_releaser_timezone_ahead_of_utc() -> None: + """Guards the Zenodo archiving citation gate against a UTC+N false rejection. + + A releaser east of UTC writes tomorrow's UTC date when tagging after local + midnight, so one day of skew must pass. + """ + citation = CitationRelease(version="0.5.1", date_released=date(2026, 1, 3)) + + validate_public_release_citation("0.5.1", citation, date(2026, 1, 2)) + + +def test_public_release_citation_rejects_future_date() -> None: + """Guards the Zenodo archiving citation gate against fabricated release dates.""" + citation = CitationRelease(version="0.5.1", date_released=date(2026, 2, 1)) + + with pytest.raises(ReleaseVersionError, match="in the future"): + validate_public_release_citation("0.5.1", citation, date(2026, 1, 2)) + + +def test_public_release_cli_rejects_stale_citation( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Guards the Zenodo archiving citation gate at the workflow boundary.""" + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text('[project]\nversion = "0.5.1"\n') + monkeypatch.setenv("GITHUB_OUTPUT", str(tmp_path / "github-output")) + + with pytest.raises(SystemExit) as excinfo: + main( + [ + "public-release", + "--pyproject", + str(pyproject), + "--citation", + str(_citation(tmp_path, version="0.5.0")), + "--tag", + "v0.5.1", + ] + ) + + assert excinfo.value.code == 1 + + +def test_internal_preview_ignores_citation_drift( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Guards `main`'s .devN line against tag-time citation rules. + + CITATION.cff names the last published release, so while main sits on a + `.devN` version the citation file is legitimately behind pyproject. + """ + pyproject = tmp_path / "pyproject.toml" + output = tmp_path / "github-output" + pyproject.write_text('[project]\nversion = "0.5.2.dev0"\n') + _citation(tmp_path, version="0.5.1") + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("GITHUB_OUTPUT", str(output)) + + assert ( + main( + [ + "internal-preview", + "--pyproject", + str(pyproject), + "--run-number", + "321", + ] + ) + == 0 + ) + + assert output.read_text() == "publish=true\nversion=0.5.2.dev321\n" diff --git a/tests/test_release_workflow_wiring.py b/tests/test_release_workflow_wiring.py index 64e5ee4cf..20d75a177 100644 --- a/tests/test_release_workflow_wiring.py +++ b/tests/test_release_workflow_wiring.py @@ -421,3 +421,47 @@ def test_preview_rejects_invalid_explicit_provenance(tmp_path: Path) -> None: assert result.returncode != 0 assert "invalid tested SHA" in result.stdout assert github_output == "" + + +def _public_release_steps() -> list[dict]: + return _workflow("public-release.yml")["jobs"]["publish"]["steps"] + + +def _step_index(steps: list[dict], name: str) -> int: + return next(index for index, step in enumerate(steps) if step.get("name") == name) + + +def test_public_release_validates_citation_before_anything_irreversible() -> None: + """Guards the Zenodo archiving citation gate against running too late. + + A stale CITATION.cff must stop the release while it is still reversible; the + PyPI publish and the GitHub Release that triggers Zenodo archiving are not. + """ + steps = _public_release_steps() + validate = _step_index(steps, "Validate release tag") + + assert validate < _step_index(steps, "Build distributions") + assert validate < _step_index(steps, "Publish public release") + assert validate < _step_index(steps, "Create or update GitHub release") + + +def test_public_release_validation_can_read_citation_yaml() -> None: + """Guards the Zenodo archiving citation gate against a missing PyYAML. + + The step runs with `--no-project`, so the release tool's YAML dependency has + to be requested explicitly or the gate dies on import. + """ + steps = _public_release_steps() + run = steps[_step_index(steps, "Validate release tag")]["run"] + + assert "tools/release_version.py public-release" in " ".join(run.split()) + assert "--with pyyaml" in run + + +def test_internal_preview_version_step_can_import_the_release_tool() -> None: + """Guards the preview lane against the release tool's YAML import.""" + preview = _workflow("internal-preview-release.yml") + steps = preview["jobs"]["publish"]["steps"] + run = steps[_step_index(steps, "Compute internal preview version")]["run"] + + assert "--with pyyaml" in run diff --git a/tools/release_version.py b/tools/release_version.py index c5d892ddb..1d4baafc9 100644 --- a/tools/release_version.py +++ b/tools/release_version.py @@ -8,10 +8,17 @@ import tomllib from collections.abc import Sequence from dataclasses import dataclass +from datetime import UTC, date, datetime, timedelta from pathlib import Path +import yaml from packaging.version import InvalidVersion, Version +# CITATION.cff names the last published release, so its `date-released` is +# written by a human whose local date can already be a day ahead of the runner's +# UTC date. Tolerate exactly that much, and nothing more. +CITATION_DATE_SKEW = timedelta(days=1) + class ReleaseVersionError(ValueError): """Raised when a release version does not match BenchFlow policy.""" @@ -25,6 +32,14 @@ class InternalPreviewDecision: version: str = "" +@dataclass(frozen=True) +class CitationRelease: + """Release identity recorded in a CITATION.cff file.""" + + version: str + date_released: date + + def read_project_version(pyproject_path: Path) -> str: """Return the `[project].version` value from a pyproject file.""" try: @@ -41,6 +56,45 @@ def read_project_version(pyproject_path: Path) -> str: return version +def read_citation_release(citation_path: Path) -> CitationRelease: + """Return the release identity recorded in a CITATION.cff file.""" + try: + citation = yaml.safe_load(citation_path.read_text(encoding="utf-8")) + except FileNotFoundError as exc: + raise ReleaseVersionError( + f"{citation_path} is missing; public releases must ship a citation " + "file naming the released version." + ) from exc + except yaml.YAMLError as exc: + raise ReleaseVersionError(f"Could not parse {citation_path} as YAML.") from exc + + if not isinstance(citation, dict): + raise ReleaseVersionError(f"{citation_path} must contain a YAML mapping.") + + return CitationRelease( + version=_citation_version(citation, citation_path), + date_released=_citation_date_released(citation, citation_path), + ) + + +def validate_public_release_citation( + version_text: str, citation: CitationRelease, today: date +) -> None: + """Validate that CITATION.cff names the version being released today.""" + if _parse_version(citation.version) != _parse_version(version_text): + raise ReleaseVersionError( + f"CITATION.cff records version {citation.version!r} but this release " + f"publishes {version_text!r}. Update CITATION.cff `version` and " + "`date-released` in the release PR before tagging." + ) + if citation.date_released > today + CITATION_DATE_SKEW: + raise ReleaseVersionError( + f"CITATION.cff records date-released {citation.date_released.isoformat()}, " + f"which is in the future (today is {today.isoformat()} UTC). A citation " + "file must not claim a release date that has not happened." + ) + + def compute_internal_preview_version( version_text: str, run_number: str | int ) -> InternalPreviewDecision: @@ -98,6 +152,38 @@ def _parse_version(version_text: str) -> Version: ) from exc +def _citation_version(citation: dict, citation_path: Path) -> str: + # YAML parses `version: 1.0` as a float and `version: 0.7.4` as a string, so + # accept either shape and let PEP 440 comparison decide equality. + version = citation.get("version") + if isinstance(version, bool) or not isinstance(version, (str, int, float)): + raise ReleaseVersionError( + f"{citation_path} must set `version` to the released version." + ) + return str(version) + + +def _citation_date_released(citation: dict, citation_path: Path) -> date: + # An unquoted `date-released: 2026-08-17` arrives as a date; a quoted one + # arrives as a string. + date_released = citation.get("date-released") + if isinstance(date_released, datetime): + return date_released.date() + if isinstance(date_released, date): + return date_released + if isinstance(date_released, str): + try: + return date.fromisoformat(date_released) + except ValueError as exc: + raise ReleaseVersionError( + f"{citation_path} has an unparsable `date-released` " + f"{date_released!r}; use an ISO 8601 date." + ) from exc + raise ReleaseVersionError( + f"{citation_path} must set `date-released` to the release date." + ) + + def _normalize_run_number(run_number: str | int) -> str: run_number_text = str(run_number) if not run_number_text.isdecimal() or int(run_number_text) <= 0: @@ -136,6 +222,11 @@ def _cmd_public_release(args: argparse.Namespace) -> int: tag_name, read_project_version(args.pyproject), ) + validate_public_release_citation( + version, + read_citation_release(args.citation), + datetime.now(UTC).date(), + ) write_github_output({"version": version}) return 0 @@ -162,7 +253,7 @@ def build_parser() -> argparse.ArgumentParser: public_release = subparsers.add_parser( "public-release", - help="Validate a public release tag against pyproject.toml.", + help="Validate a public release tag against pyproject.toml and CITATION.cff.", ) public_release.add_argument( "--pyproject", @@ -170,6 +261,12 @@ def build_parser() -> argparse.ArgumentParser: default=Path("pyproject.toml"), help="Path to pyproject.toml.", ) + public_release.add_argument( + "--citation", + type=Path, + default=Path("CITATION.cff"), + help="Path to CITATION.cff.", + ) public_release.add_argument( "--tag", help="Release tag name. Defaults to TAG_NAME.",