diff --git a/README.md b/README.md index 34fc913..1d1099f 100644 --- a/README.md +++ b/README.md @@ -240,7 +240,7 @@ This override applies to all vLLM models in the run. For remote providers (OpenA ## 📊 Supported Tasks -Task names follow [LMHarness](https://github.com/EleutherAI/lm-evaluation-harness) conventions. Generate+judge tasks produce pairwise preferences between two models; ELO tasks (`elo-*`) estimate a single model's ELO rating against human-annotated arena opponents. +Task names follow [LMHarness](https://github.com/EleutherAI/lm-evaluation-harness) conventions. Generate+judge tasks produce pairwise preferences between two models; tasks using the ELO protocol estimate a single model's rating against human-annotated arena opponents. ### Generate + judge (pairwise) @@ -289,7 +289,7 @@ For m-Arena-Hard, baseline completions are tied to the benchmark release: JudgeArena can estimate the ELO rating of a model by running it against opponents sampled from a human preference arena (`LMArena-100k`, `LMArena-140k`, or `ComparIA`). The LLM judge scores each battle, and the resulting ratings are computed using the Bradley-Terry model anchored against the human-annotated arena leaderboard. -Pass an `elo-` value to `--task` to trigger the ELO flow. ELO tasks take a single `--model.name` whose opponents are sampled from the arena (matching the pairwise CLI shape; `--model.baseline` is reserved for a future extension). +Select one of the packaged ELO tasks with `--task`. Its task definition selects the ELO runner, canonical arena, pinned battle sources, judge defaults, and Bradley-Terry scoring adapter. ELO tasks take a single `--model.name`; experiment settings such as sampling, filtering, bootstrapping, and calibration remain runtime options. `--model.baseline` is not used because opponents come from the selected arena. ### Quick start @@ -305,7 +305,7 @@ judgearena \ | Flag | Default | Description | |---|---|---| -| `--task elo-` | *(required)* | Arena to sample opponents from: `elo-lmarena-100k`, `elo-lmarena-140k`, `elo-lmarena`, or `elo-comparia` | +| `--task` | *(required)* | Packaged ELO task: `elo-lmarena-100k`, `elo-lmarena-140k`, `elo-lmarena`, or `elo-comparia` | | `--model.name` | *(required)* | Model under evaluation (same format as pairwise tasks) | | `--judge.model` | *(required)* | LLM judge (same format as pairwise tasks) | | `--generation.n_instructions` | all | Number of arena battles to use for evaluation | diff --git a/configs/elo_comparia.yaml b/configs/elo_comparia.yaml index c21730b..728139a 100644 --- a/configs/elo_comparia.yaml +++ b/configs/elo_comparia.yaml @@ -1,7 +1,7 @@ # Example JudgeArena ELO-rating run config. # Run with: judgearena --config_path configs/elo_comparia.yaml # ELO tasks rate a single model (model.name) against arena opponents; the -# arena is derived from the elo-* task, so elo.arena does not need to be set. +# arena and pinned battle sources come from the packaged task definition. task: elo-comparia model: name: Together/meta-llama/Llama-3.3-70B-Instruct-Turbo diff --git a/judgearena/arenas_utils.py b/judgearena/arenas_utils.py index ec2754e..5544af2 100644 --- a/judgearena/arenas_utils.py +++ b/judgearena/arenas_utils.py @@ -1,43 +1,73 @@ import warnings +from collections.abc import Mapping from pathlib import Path import pandas as pd from fast_langdetect import detect_language from huggingface_hub import snapshot_download -from judgearena.dataset_revisions import hf_revision from judgearena.log import get_logger +from judgearena.tasks.schema import HuggingFaceDatasetSource logger = get_logger(__name__) +def _download_arena_dataset( + *, + repo_id: str, + default_allow_patterns: str | tuple[str, ...], + dataset_sources: Mapping[str, HuggingFaceDatasetSource], +) -> str: + """Download one arena source at the revision pinned by its task definition.""" + try: + source = dataset_sources[repo_id] + except KeyError as exc: + raise ValueError( + f"Arena task does not declare required dataset source {repo_id!r}." + ) from exc + return snapshot_download( + repo_id=repo_id, + repo_type="dataset", + allow_patterns=source.allow_patterns or default_allow_patterns, + force_download=False, + revision=source.revision, + ) + + def _extract_instruction_text(turn: dict) -> str: - """Extract plain instruction text from a conversation first turn. + """Extract plain instruction text from a conversation turn. Handles both the 100k schema (content is a plain string) and the 140k - schema (content is an array of {type, text, ...} objects). + schema (content is an array of {type, text, ...} objects). Moderated or + empty turns ship ``content: None`` and yield an empty string. """ - content = turn["content"] + content = turn.get("content") + if content is None: + return "" if isinstance(content, str): return content - return " ".join(block["text"] for block in content if block.get("type") == "text") + return " ".join( + block.get("text") or "" + for block in content + if isinstance(block, dict) and block.get("type") == "text" + ) KNOWN_ARENAS = ["LMArena-100k", "LMArena-55k", "LMArena-140k", "ComparIA"] def _load_arena_dataframe( - arena: str, comparia_revision: str | None = None + arena: str, + *, + dataset_sources: Mapping[str, HuggingFaceDatasetSource], ) -> pd.DataFrame: assert arena in KNOWN_ARENAS if arena == "LMArena-55k": repo_id = "lmarena-ai/arena-human-preference-55k" - path = snapshot_download( + path = _download_arena_dataset( repo_id=repo_id, - repo_type="dataset", - allow_patterns="*.csv", - force_download=False, - revision=hf_revision(repo_id), + default_allow_patterns="*.csv", + dataset_sources=dataset_sources, ) df = pd.read_csv(Path(path) / "train.csv") @@ -74,12 +104,10 @@ def _winner_55k(row) -> str | None: elif "LMArena" in arena: size = arena.split("-")[1] # "100k" or "140k" repo_id = f"lmarena-ai/arena-human-preference-{size}" - path = snapshot_download( + path = _download_arena_dataset( repo_id=repo_id, - repo_type="dataset", - allow_patterns="*parquet", - force_download=False, - revision=hf_revision(repo_id), + default_allow_patterns="*parquet", + dataset_sources=dataset_sources, ) parquet_files = sorted((Path(path) / "data").glob("*.parquet")) df = pd.concat([pd.read_parquet(f) for f in parquet_files], ignore_index=True) @@ -101,12 +129,10 @@ def _winner_55k(row) -> str | None: df["benchmark"] = arena else: - path = snapshot_download( + path = _download_arena_dataset( repo_id="ministere-culture/comparia-votes", - repo_type="dataset", - allow_patterns="*", - revision=comparia_revision, - force_download=False, + default_allow_patterns="*", + dataset_sources=dataset_sources, ) df = pd.read_parquet(Path(path) / "votes.parquet") @@ -176,18 +202,16 @@ def get_winner( return df -_DEFAULT_COMPARIA_REVISION = hf_revision("ministere-culture/comparia-votes") - - def load_arena_dataframe( arena: str | None, - comparia_revision: str | None = _DEFAULT_COMPARIA_REVISION, + *, + dataset_sources: Mapping[str, HuggingFaceDatasetSource], ) -> pd.DataFrame: """Load battles from one or all arenas. :param arena: one of "LMArena-100k", "LMArena-140k", "ComparIA", "LMArena" (concatenation of both LMArena variants), or None (all arenas). - :param comparia_revision: pinned revision for the ComparIA dataset. + :param dataset_sources: pinned sources declared by the task, keyed by repo ID. :return: dataframe containing battles for the arena(s) selected. """ if arena is None: @@ -195,17 +219,24 @@ def load_arena_dataframe( elif arena == "LMArena": arenas = ["LMArena-100k", "LMArena-55k", "LMArena-140k"] else: - return _load_arena_dataframe(arena, comparia_revision) + return _load_arena_dataframe(arena, dataset_sources=dataset_sources) return pd.concat( - [_load_arena_dataframe(a, comparia_revision) for a in arenas], + [_load_arena_dataframe(a, dataset_sources=dataset_sources) for a in arenas], ignore_index=True, ) def main(): - for arena in KNOWN_ARENAS: - logger.info("Loading %s", arena) - df = _load_arena_dataframe(arena) + from judgearena.datasets import load_battles + from judgearena.tasks.registry import load_tasks + from judgearena.tasks.schema import EloProtocol + + for task_id, task in load_tasks().items(): + if not isinstance(task.spec.protocol, EloProtocol): + continue + logger.info("Loading %s", task_id) + df = load_battles(task) + arena = task.spec.protocol.arena n_battles = len(df) n_models = len(set(df["model_a"]) | set(df["model_b"])) n_languages = df["lang"].nunique() diff --git a/judgearena/benchmarks/elo/runner.py b/judgearena/benchmarks/elo/runner.py index db7efcd..0ddd5eb 100644 --- a/judgearena/benchmarks/elo/runner.py +++ b/judgearena/benchmarks/elo/runner.py @@ -7,7 +7,7 @@ import numpy as np import pandas as pd -from judgearena.arenas_utils import _extract_instruction_text, load_arena_dataframe +from judgearena.arenas_utils import _extract_instruction_text from judgearena.artifacts import ( prepare_run_directory, safe_filename, @@ -16,13 +16,14 @@ from judgearena.battles import Leaderboard, summarize_bootstrap, write_battles from judgearena.benchmarks.elo.rating import ( arena_anchor_battles, - fit_bradley_terry, prefs_to_battle_results, sampling_cache_token, select_seeded_random_arena_battles, winner_to_pref, ) +from judgearena.benchmarks.elo.scoring import ELO_SCORERS from judgearena.benchmarks.execution import build_generation_kwargs +from judgearena.datasets import load_battles from judgearena.evaluate import ( PairScore, calibrate_temperature, @@ -33,6 +34,7 @@ from judgearena.generate import generate_instructions from judgearena.log import get_logger from judgearena.models import build_default_judge_model_kwargs, make_model +from judgearena.tasks.schema import EloProtocol, ResolvedTaskSpec from judgearena.utils import cache_function_dataframe, compute_pref_summary from judgearena.utils.eval import PrefSummary, Report @@ -135,19 +137,42 @@ def render(self) -> None: print("\n No overlapping arena models to compute MAE.") -def main(cfg: "RunConfig") -> dict: - assert cfg.elo is not None # main is dispatched only for elo tasks +def run_elo(cfg: "RunConfig", task: ResolvedTaskSpec | None = None) -> dict: + """Rate one model against the human battles defined by an ELO task.""" + protocol = task.spec.protocol if task is not None else None + if not isinstance(protocol, EloProtocol): + raise ValueError(f"Task {cfg.task!r} does not define an ELO protocol.") + if cfg.elo is None: + raise ValueError(f"Task {cfg.task!r} requires ELO runtime settings.") + arena = protocol.arena + scorer = ELO_SCORERS[protocol.scoring.adapter] run_started_at = datetime.now(UTC) rng = np.random.default_rng(cfg.run.seed) # Step 1: Load arena battles - logger.info("Step 1: Loading battles from %s", cfg.elo.arena) - df_arena_all = load_arena_dataframe(arena=cfg.elo.arena) + logger.info("Step 1: Loading battles from %s", arena) + df_arena_all = load_battles(task) + + # Filter by language: a task variant (e.g. elo-lmarena-140k-en) preselects + # languages; elo.languages narrows further within that selection. + selected_languages = list(cfg.elo.languages or []) + if task.selection is not None: + variant_languages = list(task.selection.values) + if selected_languages: + selected_languages = [ + lang for lang in selected_languages if lang in set(variant_languages) + ] + if not selected_languages: + raise ValueError( + f"elo.languages {cfg.elo.languages} has no overlap with the " + f"languages of task {cfg.task!r} ({variant_languages})." + ) + else: + selected_languages = variant_languages - # Filter by language if specified df_battles = df_arena_all - if cfg.elo.languages: - df_battles = df_battles[df_battles["lang"].isin(cfg.elo.languages)] + if selected_languages: + df_battles = df_battles[df_battles["lang"].isin(selected_languages)] random_sampling = cfg.elo.elo_random_battles is not None sampling_metadata: dict[str, object] = {"sampling_mode": "head"} @@ -211,7 +236,9 @@ def main(cfg: "RunConfig") -> dict: def replace_slash(s: str) -> str: return s.replace("/", "_") - languages_str = "-".join(sorted(cfg.elo.languages)) if cfg.elo.languages else "all" + languages_str = ( + "-".join(sorted(selected_languages)) if selected_languages else "all" + ) extra_kwargs_str = ( "_".join(f"{k}={v}" for k, v in sorted(extra_kwargs.items())) if extra_kwargs @@ -223,7 +250,7 @@ def replace_slash(s: str) -> str: n_instructions_per_language=cfg.elo.n_instructions_per_language, ) cache_suffix = ( - f"{cfg.elo.arena}_{replace_slash(cfg.model.name)}_" + f"{arena}_{replace_slash(cfg.model.name)}_" f"{cache_token}_" f"{languages_str}_{cfg.generation.truncate_all_input_chars}_{extra_kwargs['max_tokens']}" + (f"_{extra_kwargs_str}" if extra_kwargs_str else "") @@ -268,7 +295,7 @@ def replace_slash(s: str) -> str: ] our_completions = completions.tolist() - resolved_prompt = resolve_run_judge_prompt(cfg.elo.arena, cfg.judge) + resolved_prompt = resolve_run_judge_prompt(cfg.task, cfg.judge) completions_A = [ our_completions[i] if our_model_is_position_a[i] else opponent_completions[i] @@ -394,7 +421,7 @@ def run_judge() -> pd.DataFrame: df_results = pd.concat([df_llm_judge, df_arena], ignore_index=True) # Compute human-only BT ratings as ground-truth reference - human_elo = fit_bradley_terry( + human_elo = scorer.fit( df_arena, pref_col="pref_hard", baseline_model=cfg.elo.baseline_model ) @@ -438,7 +465,6 @@ def run_judge() -> pd.DataFrame: judge_chat_model_cal = make_model( model=cfg.judge.model, - max_tokens=cfg.judge.max_out_tokens, **judge_extra_kwargs, ) cal_annotations, _, cal_prefs = judge_and_parse_prefs( @@ -540,7 +566,7 @@ def run_judge() -> pd.DataFrame: df_sample = df_results.sample( n=len(df_results), replace=True, random_state=int(rng.integers(0, 2**31)) ) - ratings = fit_bradley_terry( + ratings = scorer.fit( df_sample, pref_col=pref_col, baseline_model=cfg.elo.baseline_model ) bootstrap_ratings.append(ratings) @@ -570,7 +596,7 @@ def run_judge() -> pd.DataFrame: ) report = EloReport( - arena=cfg.elo.arena, + arena=arena, judge_model=cfg.judge.model, summary=summary, num_battles=n, @@ -597,17 +623,17 @@ def run_judge() -> pd.DataFrame: res_dir = prepare_run_directory( cfg, Path(cfg.run.result_folder) - / f"elo-{safe_filename(cfg.elo.arena)}-{safe_filename(model_name)}-" + / f"elo-{safe_filename(arena)}-{safe_filename(model_name)}-" f"{safe_filename(cfg.judge.model)}", ) result_path = report.save(res_dir / f"results-{safe_filename(model_name)}.json") # Persist only the run's own llm-judge battles (a few KB). The human arena # anchors are identical across every run, so we do not duplicate them per - # experiment — recompute ELO by recombining with - # arena_anchor_battles(load_arena_dataframe(arena)). question_id is the - # instruction-index join key back to the arena initial table / completion - # cache. battles.parquet keeps pref_hard so both hard and soft ELO recompute. + # experiment — recompute ELO by loading this task's pinned battles again and + # applying arena_anchor_battles(). question_id is the join key back to the + # arena table / completion cache. battles.parquet keeps pref_hard so both + # hard and soft ELO can be recomputed. battle_cols = [ "model_a", "model_b", @@ -627,7 +653,7 @@ def run_judge() -> pd.DataFrame: res_dir / "bootstrap_ratings.csv", index=False ) Leaderboard( - arena=cfg.elo.arena, + arena=arena, model=model_name, judge_model=cfg.judge.model, n_bootstraps=n_bootstraps, @@ -640,7 +666,7 @@ def run_judge() -> pd.DataFrame: # a metadata-write failure should not sink an already-completed run. write_run_metadata_safely( output_dir=res_dir, - entrypoint="judgearena.benchmarks.elo.runner.main", + entrypoint="judgearena.benchmarks.elo.runner.run_elo", run=cfg.model_dump(), results=results, input_payloads=( diff --git a/judgearena/benchmarks/elo/scoring.py b/judgearena/benchmarks/elo/scoring.py new file mode 100644 index 0000000..af6c01d --- /dev/null +++ b/judgearena/benchmarks/elo/scoring.py @@ -0,0 +1,28 @@ +"""Runtime scoring adapters for ELO rating tasks. + +A scorer turns judged battles into the metric a benchmark reports. Each +protocol defines its own, so scorers are named components selected by task +YAML rather than logic in the runner: the runner produces preferences, the +scorer owns the metric math. +""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass + +from judgearena.benchmarks.elo.rating import fit_bradley_terry + +RatingFunction = Callable[..., dict[str, float]] + + +@dataclass(frozen=True) +class EloScorer: + """Rating implementation selected by an ELO task's scoring adapter.""" + + fit: RatingFunction + + +ELO_SCORERS = { + "bradley_terry": EloScorer(fit=fit_bradley_terry), +} diff --git a/judgearena/benchmarks/registry.py b/judgearena/benchmarks/registry.py index 83dc514..a7249af 100644 --- a/judgearena/benchmarks/registry.py +++ b/judgearena/benchmarks/registry.py @@ -44,10 +44,12 @@ class ResolvedBenchmark: def benchmark_adapters() -> tuple[BenchmarkAdapter, ...]: """Return registered benchmark implementations, specific first.""" + from judgearena.benchmarks.elo.runner import run_elo from judgearena.benchmarks.mt_bench.runner import run_mt_bench_benchmark from judgearena.benchmarks.pairwise.runner import run_pairwise return ( + BenchmarkAdapter("elo", frozenset(), run_elo), BenchmarkAdapter("mt_bench", frozenset(), run_mt_bench_benchmark), BenchmarkAdapter("pairwise", None, run_pairwise), ) diff --git a/judgearena/cli.py b/judgearena/cli.py index 4a7d339..7a306f1 100644 --- a/judgearena/cli.py +++ b/judgearena/cli.py @@ -10,10 +10,8 @@ from pydantic import ValidationError -from judgearena.benchmarks.elo.runner import main as main_elo from judgearena.benchmarks.runner import run_benchmark from judgearena.config import build_run_config -from judgearena.constants import ELO_TASK_PREFIX from judgearena.log import configure_logging, get_logger logger = get_logger(__name__) @@ -46,10 +44,7 @@ def cli(argv: list[str] | None = None) -> None: configure_logging(cfg.run.verbosity, log_file=cfg.run.log_file) logger.debug("Running with config: %s", cfg.model_dump()) - if cfg.task.startswith(ELO_TASK_PREFIX): - main_elo(cfg) - else: - run_benchmark(cfg) + run_benchmark(cfg) if __name__ == "__main__": diff --git a/judgearena/config.py b/judgearena/config.py index 79479a7..8c213cc 100644 --- a/judgearena/config.py +++ b/judgearena/config.py @@ -17,8 +17,9 @@ ) from judgearena.benchmarks.pairwise.baselines import native_pairwise_baseline -from judgearena.constants import ELO_TASK_PREFIX, ELO_TASK_TO_ARENA +from judgearena.datasets.fluency import is_fluency_task from judgearena.tasks.registry import get_packaged_task +from judgearena.tasks.schema import EloProtocol # Set by build_run_config() for the duration of RunConfig() construction. _ACTIVE_CONFIG_PATH: str | None = None @@ -286,13 +287,13 @@ class GenerationArgs(BaseModel): class EloArgs(BaseModel): - """Settings specific to elo-rating tasks (``--task elo-*``).""" + """Experiment settings for tasks using the ELO protocol.""" model_config = ConfigDict(use_attribute_docstrings=True) arena: str | None = None - """Arena identifier whose battles supply the opponents. Derived from the - ``elo-*`` task when left unset.""" + """Arena whose battles supply opponents. Derived from the task definition; + an explicit value must match it.""" baseline_model: str | None = None """Model anchored at 1000 ELO; ratings are reported relative to it.""" @@ -367,8 +368,8 @@ class RunConfig(BaseSettings): ) task: str - """Benchmark task ID. Use ``judgearena tasks list`` for packaged tasks; - legacy ELO task IDs use the ``elo-*`` prefix.""" + """Packaged benchmark task ID. Use ``judgearena tasks list`` to inspect + available definitions.""" model: ModelArgs = Field(default_factory=ModelArgs) """Model(s) under evaluation and their generation settings.""" @@ -380,7 +381,7 @@ class RunConfig(BaseSettings): """Instruction count and input truncation.""" elo: EloArgs | None = None - """ELO-task settings (only for ``elo-*`` tasks).""" + """Runtime settings used only by tasks with an ELO protocol.""" run: RunArgs = Field(default_factory=RunArgs) """Run-level settings (seed, output, caching, logging).""" @@ -388,50 +389,68 @@ class RunConfig(BaseSettings): @model_validator(mode="after") def _validate(self) -> RunConfig: resolved_task = get_packaged_task(self.task) - if resolved_task is not None: - task_judge = resolved_task.spec.protocol.judge - if "swap_mode" not in self.judge.model_fields_set: - self.judge.swap_mode = task_judge.default_swap_mode - if self.judge.swap_mode not in task_judge.allowed_swap_modes: + if resolved_task is None: + if not is_fluency_task(self.task): raise ValueError( - f"judge.swap_mode={self.judge.swap_mode!r} is not supported " - f"by task {self.task!r}; choose from " - f"{list(task_judge.allowed_swap_modes)}." - ) - if ( - self.judge.temperature is None - and task_judge.default_temperature is not None - ): - self.judge.temperature = task_judge.default_temperature - - baseline = resolved_task.spec.protocol.baseline - if ( - self.model.baseline is not None - and getattr(baseline, "allow_runtime_override", True) is False - ): - raise ValueError( - f"model.baseline cannot override the baseline defined by " - f"task {self.task!r}." + f"Unknown task {self.task!r}; use 'judgearena tasks list' to " + "inspect packaged tasks." ) + # Fluency tasks are not packaged yet and run through the legacy path. + if self.elo is not None: + raise ValueError("elo config is only valid for ELO tasks.") + if self.model.name is None: + raise ValueError("model.name is required.") + if self.model.baseline is None: + raise ValueError(f"model.baseline is required for task {self.task!r}.") + return self + + protocol = resolved_task.spec.protocol + task_judge = protocol.judge + if "swap_mode" not in self.judge.model_fields_set: + self.judge.swap_mode = task_judge.default_swap_mode + if self.judge.swap_mode not in task_judge.allowed_swap_modes: + raise ValueError( + f"judge.swap_mode={self.judge.swap_mode!r} is not supported " + f"by task {self.task!r}; choose from " + f"{list(task_judge.allowed_swap_modes)}." + ) + if ( + self.judge.temperature is None + and task_judge.default_temperature is not None + ): + self.judge.temperature = task_judge.default_temperature + + baseline = protocol.baseline + if ( + self.model.baseline is not None + and getattr(baseline, "allow_runtime_override", True) is False + ): + raise ValueError( + f"model.baseline cannot override the baseline defined by " + f"task {self.task!r}." + ) - is_elo = self.task.startswith(ELO_TASK_PREFIX) + is_elo = isinstance(protocol, EloProtocol) if is_elo: if self.elo is None: self.elo = EloArgs() - if self.elo.arena is None: - if self.task not in ELO_TASK_TO_ARENA: - raise ValueError( - f"Unknown elo task {self.task!r}; expected one of " - f"{list(ELO_TASK_TO_ARENA)}." - ) - self.elo.arena = ELO_TASK_TO_ARENA[self.task] + if self.elo.arena is not None and self.elo.arena != protocol.arena: + raise ValueError( + f"elo.arena={self.elo.arena!r} does not match task " + f"{self.task!r} ({protocol.arena!r})." + ) + self.elo.arena = protocol.arena + if "soft_elo" not in self.elo.model_fields_set: + self.elo.soft_elo = protocol.scoring.default_soft + if "soft_elo_temperature" not in self.elo.model_fields_set: + self.elo.soft_elo_temperature = protocol.scoring.default_temperature if self.model.name is None: - raise ValueError("model.name is required for elo tasks.") + raise ValueError("model.name is required for ELO tasks.") if self.model.baseline is not None: - raise ValueError("model.baseline is not supported for elo tasks.") + raise ValueError("model.baseline is not supported for ELO tasks.") else: if self.elo is not None: - raise ValueError("elo config is only valid for elo-* tasks.") + raise ValueError("elo config is only valid for ELO tasks.") if self.model.name is None: raise ValueError("model.name is required.") if ( diff --git a/judgearena/constants.py b/judgearena/constants.py index 9237f42..2c1e9bd 100644 --- a/judgearena/constants.py +++ b/judgearena/constants.py @@ -1,22 +1,7 @@ -"""Project-wide constants: task identifiers and task→arena mapping.""" +"""Project-wide inference constants.""" from __future__ import annotations -ELO_TASK_PREFIX = "elo-" -"""Prefix marking a task as an ELO-rating run (e.g. ``elo-lmarena-100k``).""" - -# Lowercase CLI task name -> canonical arena identifier used inside -# the arena loaders and the ``benchmark`` column of saved battle dataframes. -# The CLI stays lowercase (matching ``alpaca-eval`` conventions) while internal -# identifiers keep their original casing. -ELO_TASK_TO_ARENA: dict[str, str] = { - "elo-lmarena-100k": "LMArena-100k", - "elo-lmarena-140k": "LMArena-140k", - "elo-lmarena": "LMArena", - "elo-comparia": "ComparIA", -} - - # vLLM reasoning markers shared by the inference layer (judgearena.models) and # the reasoning-tag stripping in judgearena.utils.text. VLLM_REASONING_START_STR = "" diff --git a/judgearena/dataset_revisions.py b/judgearena/dataset_revisions.py index 481124c..2641a9d 100644 --- a/judgearena/dataset_revisions.py +++ b/judgearena/dataset_revisions.py @@ -16,18 +16,9 @@ # do not yet have a stable pin and is recorded as such in the metadata so # the gap is visible. HF_DATASET_REVISIONS: dict[str, str | None] = { - # LMArena human-preference battles - "lmarena-ai/arena-human-preference-100k": "72e85b3ddc9c81bf7b659d6b03d4126dfd8fb34a", - "lmarena-ai/arena-human-preference-140k": "6322995ab34d7c2693e3f47dd13fa5caa0789a74", - "lmarena-ai/arena-human-preference-55k": "18c298340948c0e7f7727399fd459cca6ce0ca6f", - # ComparIA (already pinned via the legacy comparia_revision argument). - "ministere-culture/comparia-votes": "7a40bce496c1f2aa3be4001da85a49cb4743042b", - # MT-Bench questions (LMSYS Space). - "lmsys/mt-bench": "a4b674ca573c24143824ac7f60d9173e7081e37d", + # Sources used by packaged tasks are pinned in their task YAML definitions. # Multilingual fluency contexts (generated by scripts/fluency/generate_fluency.py). "geoalgo/multilingual-fluency": "ea6acc306273cc9cfa49bf5029e6fee74e22b985", - # Arena-Hard official source (used via datasets.load_dataset). - "lmarena-ai/arena-hard-auto": "15f3746e21432264ce9b453999bde4f3c946d2e6", } diff --git a/judgearena/datasets/__init__.py b/judgearena/datasets/__init__.py index 961f255..54578c7 100644 --- a/judgearena/datasets/__init__.py +++ b/judgearena/datasets/__init__.py @@ -1,32 +1,56 @@ import pandas as pd from judgearena.log import get_logger -from judgearena.tasks.registry import get_packaged_task +from judgearena.tasks.schema import ResolvedTaskSpec logger = get_logger(__name__) -def load_instructions(dataset: str, n_instructions: int | None = None) -> pd.DataFrame: - resolved_task = get_packaged_task(dataset) - if resolved_task is not None: - from judgearena import utils as judgearena_utils - from judgearena.datasets.registry import resolve_dataset_adapter +def _resolve_task(dataset: str | ResolvedTaskSpec) -> ResolvedTaskSpec: + if not isinstance(dataset, str): + return dataset + from judgearena.tasks.registry import get_packaged_task - adapter = resolve_dataset_adapter(resolved_task.spec.dataset.adapter) - df_instructions = adapter.load_instructions( - resolved_task, judgearena_utils.data_root / "tables" - ) + resolved = get_packaged_task(dataset) + if resolved is None: + raise ValueError(f"Unsupported task dataset {dataset!r}.") + return resolved - else: - raise ValueError(f"Unsupported instruction dataset {dataset!r}.") + +def load_instructions( + dataset: str | ResolvedTaskSpec, n_instructions: int | None = None +) -> pd.DataFrame: + """Load instructions by task ID or an already-resolved task definition.""" + resolved_task = _resolve_task(dataset) + from judgearena import utils as judgearena_utils + from judgearena.datasets.registry import resolve_dataset_adapter + + adapter = resolve_dataset_adapter(resolved_task.spec.dataset.adapter) + df_instructions = adapter.load_instructions( + resolved_task, judgearena_utils.data_root / "tables" + ) df_instructions = df_instructions.set_index("instruction_index").sort_index() - logger.info("Loaded %d instructions for %s.", len(df_instructions), dataset) + logger.info( + "Loaded %d instructions for %s.", len(df_instructions), resolved_task.task + ) if n_instructions is None: n_instructions = len(df_instructions) return df_instructions.head(n_instructions) +def load_battles(dataset: str | ResolvedTaskSpec) -> pd.DataFrame: + """Load human preference battles for a battle-backed task.""" + resolved_task = _resolve_task(dataset) + from judgearena import utils as judgearena_utils + from judgearena.datasets.registry import resolve_battle_dataset_adapter + + adapter = resolve_battle_dataset_adapter(resolved_task.spec.dataset.adapter) + battles = adapter.load_battles(resolved_task, judgearena_utils.data_root / "tables") + logger.info("Loaded %d battles for %s.", len(battles), resolved_task.task) + return battles + + if __name__ == "__main__": instructions = load_instructions(dataset="alpaca-eval") print(instructions) diff --git a/judgearena/datasets/arena_battles.py b/judgearena/datasets/arena_battles.py new file mode 100644 index 0000000..a9a0bc7 --- /dev/null +++ b/judgearena/datasets/arena_battles.py @@ -0,0 +1,56 @@ +"""Dataset adapter for human preference battles used by ELO tasks.""" + +from __future__ import annotations + +from pathlib import Path + +import pandas as pd +from huggingface_hub import snapshot_download + +from judgearena.arenas_utils import ( + KNOWN_ARENAS, + load_arena_dataframe, +) +from judgearena.tasks.schema import ( + EloProtocol, + HuggingFaceDatasetSource, + ResolvedTaskSpec, +) + + +def _task_sources( + task: ResolvedTaskSpec, +) -> tuple[EloProtocol, dict[str, HuggingFaceDatasetSource]]: + protocol = task.spec.protocol + if not isinstance(protocol, EloProtocol): + raise ValueError(f"Task {task.task!r} does not define an ELO protocol.") + if protocol.arena not in {*KNOWN_ARENAS, "LMArena"}: + raise ValueError(f"Unsupported ELO arena {protocol.arena!r}.") + + sources: dict[str, HuggingFaceDatasetSource] = {} + for source in task.spec.dataset.sources.values(): + if not isinstance(source, HuggingFaceDatasetSource): + raise ValueError("ELO arena sources must be Hugging Face datasets.") + if source.repo_id in sources: + raise ValueError(f"Duplicate ELO arena source {source.repo_id!r}.") + sources[source.repo_id] = source + return protocol, sources + + +def download_task_sources(task: ResolvedTaskSpec, _local_dir: Path) -> None: + """Download every pinned human-battle source declared by the task.""" + _, sources = _task_sources(task) + for source in sources.values(): + snapshot_download( + repo_id=source.repo_id, + repo_type="dataset", + revision=source.revision, + allow_patterns=source.allow_patterns or None, + force_download=False, + ) + + +def load_task_battles(task: ResolvedTaskSpec, _local_dir: Path) -> pd.DataFrame: + """Load and normalize the task's pinned human preference battles.""" + protocol, sources = _task_sources(task) + return load_arena_dataframe(protocol.arena, dataset_sources=sources) diff --git a/judgearena/datasets/pairwise.py b/judgearena/datasets/pairwise.py index f9c4e4c..f0a1aa1 100644 --- a/judgearena/datasets/pairwise.py +++ b/judgearena/datasets/pairwise.py @@ -69,21 +69,20 @@ def load_pairwise_task_data( raise ValueError(f"Task {task.task!r} does not use the pairwise protocol.") tables_path = local_tables_path or data_root / "tables" - adapter = resolve_dataset_adapter(task.spec.dataset.adapter) + adapter_id = task.spec.dataset.adapter + adapter = resolve_dataset_adapter(adapter_id) instructions = adapter.load_instructions(task, tables_path) if "instruction_index" in instructions.columns: instructions = instructions.set_index("instruction_index") if instructions.index.name != "instruction_index": raise ValueError( - f"Dataset adapter {adapter.name!r} must provide 'instruction_index'." + f"Dataset adapter {adapter_id!r} must provide 'instruction_index'." ) if "instruction" not in instructions.columns: - raise ValueError( - f"Dataset adapter {adapter.name!r} must provide 'instruction'." - ) + raise ValueError(f"Dataset adapter {adapter_id!r} must provide 'instruction'.") if instructions.index.has_duplicates: raise ValueError( - f"Dataset adapter {adapter.name!r} returned duplicate instruction IDs." + f"Dataset adapter {adapter_id!r} returned duplicate instruction IDs." ) instructions = instructions.sort_index() diff --git a/judgearena/datasets/registry.py b/judgearena/datasets/registry.py index 9c9016f..2642041 100644 --- a/judgearena/datasets/registry.py +++ b/judgearena/datasets/registry.py @@ -12,51 +12,77 @@ TaskDataFunction = Callable[[ResolvedTaskSpec, Path], pd.DataFrame | None] TaskDownloadFunction = Callable[[ResolvedTaskSpec, Path], None] +TaskBattleFunction = Callable[[ResolvedTaskSpec, Path], pd.DataFrame] @dataclass(frozen=True) -class DatasetAdapter: - name: str +class InstructionDatasetAdapter: + """Dataset operations required by instruction-based benchmark runners.""" + download: TaskDownloadFunction load_instructions: TaskDataFunction load_model_outputs: TaskDataFunction -def dataset_adapters() -> tuple[DatasetAdapter, ...]: - """Return registered dataset implementations.""" +@dataclass(frozen=True) +class BattleDatasetAdapter: + """Dataset operations required by arena-battle benchmark runners.""" + + download: TaskDownloadFunction + load_battles: TaskBattleFunction + + +TaskDatasetAdapter = InstructionDatasetAdapter | BattleDatasetAdapter + + +def _instruction_datasets() -> dict[str, InstructionDatasetAdapter]: from judgearena.datasets import arena_hard, judgearena_tables, m_arenahard, mt_bench - return ( - DatasetAdapter( - "judgearena_tables", + return { + "judgearena_tables": InstructionDatasetAdapter( judgearena_tables.download_task_sources, judgearena_tables.load_task_instructions, judgearena_tables.load_task_model_outputs, ), - DatasetAdapter( - "arena_hard", + "arena_hard": InstructionDatasetAdapter( arena_hard.download_task_sources, arena_hard.load_task_instructions, arena_hard.load_task_model_outputs, ), - DatasetAdapter( - "m_arena_hard", + "m_arena_hard": InstructionDatasetAdapter( m_arenahard.download_task_sources, m_arenahard.load_task_instructions, m_arenahard.load_task_model_outputs, ), - DatasetAdapter( - "mt_bench", + "mt_bench": InstructionDatasetAdapter( mt_bench.download_task_sources, mt_bench.load_task_instructions, mt_bench.load_task_model_outputs, ), - ) + } + + +def _battle_datasets() -> dict[str, BattleDatasetAdapter]: + from judgearena.datasets import arena_battles + + return { + "arena_battles": BattleDatasetAdapter( + arena_battles.download_task_sources, + arena_battles.load_task_battles, + ), + } + + +def resolve_dataset_adapter(name: str) -> InstructionDatasetAdapter: + """Return the instruction-dataset implementation registered under ``name``.""" + return _instruction_datasets()[name] + + +def resolve_battle_dataset_adapter(name: str) -> BattleDatasetAdapter: + """Return the battle-dataset implementation registered under ``name``.""" + return _battle_datasets()[name] -def resolve_dataset_adapter(name: str) -> DatasetAdapter: - """Return the implementation registered under ``name``.""" - for adapter in dataset_adapters(): - if adapter.name == name: - return adapter - raise ValueError(f"Unknown task dataset adapter {name!r}.") +def resolve_download_adapter(name: str) -> TaskDatasetAdapter: + """Return any registered dataset adapter for source prefetching.""" + return {**_instruction_datasets(), **_battle_datasets()}[name] diff --git a/judgearena/tasks/README.md b/judgearena/tasks/README.md index 9ef9191..4f5fe54 100644 --- a/judgearena/tasks/README.md +++ b/judgearena/tasks/README.md @@ -88,17 +88,18 @@ scoring adapter owns the calculation, primary metric, and metric direction; task YAML only selects those components by ID. If an upstream dataset has a new format, implement a dataset adapter under -`judgearena/datasets/` and register its ID in both the dataset registry and the -task adapter catalog. The adapter must return JudgeArena's canonical columns; -the common runner should never check a task name. +`judgearena/datasets/` and register it in the dataset registry. Task validation +derives the available adapter IDs from that registry. The adapter must return +JudgeArena's canonical columns; the common runner should never check a task +name. If a benchmark needs a genuinely different evaluation algorithm: 1. Add its typed protocol under `schema/`. 2. Add that protocol to `ProtocolSpec` in `schema/task.py`. 3. Implement and register its benchmark runner. -4. Keep algorithm-specific fields in that protocol, as MT-Bench does in - `MTBenchProtocol` and `MTBenchJudgeSpec`. +4. Keep algorithm-specific fields in that protocol, as MT-Bench and ELO do in + `MTBenchProtocol` and `EloProtocol`. Adding a schema field makes it valid in task YAML; it does not automatically make it a run-config or CLI option. Runtime overrides must be explicitly added diff --git a/judgearena/tasks/definitions/elo/_base.yaml b/judgearena/tasks/definitions/elo/_base.yaml new file mode 100644 index 0000000..39a112e --- /dev/null +++ b/judgearena/tasks/definitions/elo/_base.yaml @@ -0,0 +1,25 @@ +schema_version: 1 +task_version: 1 +tags: [elo, arena, rating] + +dataset: + adapter: arena_battles + fields: + id: question_id + instruction: conversation_a + category: lang + +protocol: + runner: elo + generation: + mode: single_turn_chat + baseline: + strategy: none + judge: + default_prompt: default + default_swap_mode: fixed + allowed_swap_modes: [fixed, both] + scoring: + adapter: bradley_terry + default_soft: true + default_temperature: 0.3 diff --git a/judgearena/tasks/definitions/elo/elo-comparia.yaml b/judgearena/tasks/definitions/elo/elo-comparia.yaml new file mode 100644 index 0000000..8809f89 --- /dev/null +++ b/judgearena/tasks/definitions/elo/elo-comparia.yaml @@ -0,0 +1,23 @@ +extends: _base.yaml +task: elo-comparia +description: Arena-anchored ELO evaluation using ComparIA human preference battles. + +variants: + selector: language + # Detected-language census of the pinned snapshot: 94% fr, 4% en; every + # other language has <500 battles. + values: [fr, en] + +dataset: + sources: + comparia: + type: huggingface_dataset + repo_id: ministere-culture/comparia-votes + revision: "7a40bce496c1f2aa3be4001da85a49cb4743042b" + allow_patterns: [votes.parquet] + +protocol: + arena: ComparIA + +metadata: + reference_implementation: https://huggingface.co/datasets/ministere-culture/comparia-votes diff --git a/judgearena/tasks/definitions/elo/elo-lmarena-100k.yaml b/judgearena/tasks/definitions/elo/elo-lmarena-100k.yaml new file mode 100644 index 0000000..1bd0c2c --- /dev/null +++ b/judgearena/tasks/definitions/elo/elo-lmarena-100k.yaml @@ -0,0 +1,21 @@ +extends: _base.yaml +task: elo-lmarena-100k +description: Arena-anchored ELO evaluation using LMSYS Chatbot Arena 100k battles. + +variants: + selector: language + values: [en, pl, ru, zh, de, ja, es, fr, ko, pt, fa, it, tr, vi, cs, ar, uk] + +dataset: + sources: + lmarena_100k: + type: huggingface_dataset + repo_id: lmarena-ai/arena-human-preference-100k + revision: "72e85b3ddc9c81bf7b659d6b03d4126dfd8fb34a" + allow_patterns: [data/*.parquet] + +protocol: + arena: LMArena-100k + +metadata: + reference_implementation: https://huggingface.co/datasets/lmarena-ai/arena-human-preference-100k diff --git a/judgearena/tasks/definitions/elo/elo-lmarena-140k.yaml b/judgearena/tasks/definitions/elo/elo-lmarena-140k.yaml new file mode 100644 index 0000000..6bc8ce4 --- /dev/null +++ b/judgearena/tasks/definitions/elo/elo-lmarena-140k.yaml @@ -0,0 +1,21 @@ +extends: _base.yaml +task: elo-lmarena-140k +description: Arena-anchored ELO evaluation using LMSYS Chatbot Arena 140k battles. + +variants: + selector: language + values: [en, pl, ru, zh, de, ja, es, fr, ko, pt, fa, it, tr, vi, cs, ar, uk] + +dataset: + sources: + lmarena_140k: + type: huggingface_dataset + repo_id: lmarena-ai/arena-human-preference-140k + revision: "6322995ab34d7c2693e3f47dd13fa5caa0789a74" + allow_patterns: [data/*.parquet] + +protocol: + arena: LMArena-140k + +metadata: + reference_implementation: https://huggingface.co/datasets/lmarena-ai/arena-human-preference-140k diff --git a/judgearena/tasks/definitions/elo/elo-lmarena.yaml b/judgearena/tasks/definitions/elo/elo-lmarena.yaml new file mode 100644 index 0000000..61c03b5 --- /dev/null +++ b/judgearena/tasks/definitions/elo/elo-lmarena.yaml @@ -0,0 +1,31 @@ +extends: _base.yaml +task: elo-lmarena +description: Arena-anchored ELO evaluation using the combined LMSYS Chatbot Arena datasets. + +variants: + selector: language + values: [en, pl, ru, zh, de, ja, es, fr, ko, pt, fa, it, tr, vi, cs, ar, uk] + +dataset: + sources: + lmarena_55k: + type: huggingface_dataset + repo_id: lmarena-ai/arena-human-preference-55k + revision: "18c298340948c0e7f7727399fd459cca6ce0ca6f" + allow_patterns: [train.csv] + lmarena_100k: + type: huggingface_dataset + repo_id: lmarena-ai/arena-human-preference-100k + revision: "72e85b3ddc9c81bf7b659d6b03d4126dfd8fb34a" + allow_patterns: [data/*.parquet] + lmarena_140k: + type: huggingface_dataset + repo_id: lmarena-ai/arena-human-preference-140k + revision: "6322995ab34d7c2693e3f47dd13fa5caa0789a74" + allow_patterns: [data/*.parquet] + +protocol: + arena: LMArena + +metadata: + reference_implementation: https://huggingface.co/lmarena-ai diff --git a/judgearena/tasks/registry.py b/judgearena/tasks/registry.py index 66e2c4f..de10d2e 100644 --- a/judgearena/tasks/registry.py +++ b/judgearena/tasks/registry.py @@ -16,10 +16,12 @@ import yaml from pydantic import ValidationError +from judgearena.benchmarks.elo.scoring import ELO_SCORERS from judgearena.benchmarks.pairwise.scoring import PAIRWISE_SCORERS from judgearena.log import get_logger from judgearena.prompts.registry import JUDGE_PROMPT_PRESETS from judgearena.tasks.schema import ( + EloProtocol, ResolvedTaskSpec, ResourceDigest, TaskProvenance, @@ -245,12 +247,14 @@ def _load_task(root: Traversable, relative_path: str) -> ResolvedTaskSpec: class AdapterCatalog: """Component IDs that task YAML files may reference.""" - runners: frozenset[str] = frozenset({"mt_bench", "pairwise"}) - datasets: frozenset[str] = frozenset( + runners: frozenset[str] = frozenset({"elo", "mt_bench", "pairwise"}) + instruction_datasets: frozenset[str] = frozenset( {"arena_hard", "judgearena_tables", "m_arena_hard", "mt_bench"} ) + battle_datasets: frozenset[str] = frozenset({"arena_battles"}) prompts: frozenset[str] = frozenset(JUDGE_PROMPT_PRESETS) - scorers: frozenset[str] = frozenset(PAIRWISE_SCORERS) + pairwise_scorers: frozenset[str] = frozenset(PAIRWISE_SCORERS) + elo_scorers: frozenset[str] = frozenset(ELO_SCORERS) def load_tasks( @@ -295,11 +299,16 @@ def _discover_tasks( def _validate_adapter_ids(resolved: ResolvedTaskSpec, adapters: AdapterCatalog) -> None: spec = resolved.spec + is_elo = isinstance(spec.protocol, EloProtocol) + scorer_names = adapters.elo_scorers if is_elo else adapters.pairwise_scorers + dataset_names = ( + adapters.battle_datasets if is_elo else adapters.instruction_datasets + ) references = { "runner": (spec.protocol.runner, adapters.runners), - "dataset adapter": (spec.dataset.adapter, adapters.datasets), + "dataset adapter": (spec.dataset.adapter, dataset_names), "prompt": (spec.protocol.judge.default_prompt, adapters.prompts), - "scorer": (spec.protocol.scoring.adapter, adapters.scorers), + "scorer": (spec.protocol.scoring.adapter, scorer_names), } for kind, (adapter_id, available) in references.items(): if adapter_id not in available: diff --git a/judgearena/tasks/schema/__init__.py b/judgearena/tasks/schema/__init__.py index 8ca0766..89b00a6 100644 --- a/judgearena/tasks/schema/__init__.py +++ b/judgearena/tasks/schema/__init__.py @@ -14,6 +14,7 @@ TaskDefaultBaseline, ) from judgearena.tasks.schema.dataset import DatasetFields, DatasetSpec +from judgearena.tasks.schema.elo import EloProtocol, EloScoringSpec from judgearena.tasks.schema.mt_bench import ( MTBenchJudgeSpec, MTBenchProtocol, @@ -50,6 +51,8 @@ "CategoryDefaultsBaseline", "DatasetFields", "DatasetSpec", + "EloProtocol", + "EloScoringSpec", "GitRawSource", "HuggingFaceDatasetSource", "HuggingFaceSpaceSource", diff --git a/judgearena/tasks/schema/elo.py b/judgearena/tasks/schema/elo.py new file mode 100644 index 0000000..08a179b --- /dev/null +++ b/judgearena/tasks/schema/elo.py @@ -0,0 +1,30 @@ +"""Schema for arena-anchored ELO rating tasks.""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import Field + +from judgearena.tasks.schema.base import StrictFrozenModel +from judgearena.tasks.schema.baselines import NoBaseline +from judgearena.tasks.schema.pairwise import PairwiseJudgeSpec, SingleTurnGeneration + + +class EloScoringSpec(StrictFrozenModel): + """Task-owned defaults for fitting arena-anchored ratings.""" + + adapter: str = Field(min_length=1) + default_soft: bool = True + default_temperature: float = Field(default=0.3, gt=0) + + +class EloProtocol(StrictFrozenModel): + """Policy used by the specialized arena battle and ELO runner.""" + + runner: Literal["elo"] + arena: str = Field(min_length=1) + generation: SingleTurnGeneration + baseline: NoBaseline + judge: PairwiseJudgeSpec + scoring: EloScoringSpec diff --git a/judgearena/tasks/schema/task.py b/judgearena/tasks/schema/task.py index 2c9713c..0b9b30a 100644 --- a/judgearena/tasks/schema/task.py +++ b/judgearena/tasks/schema/task.py @@ -12,11 +12,12 @@ OfficialOutputsBaseline, ) from judgearena.tasks.schema.dataset import DatasetSpec +from judgearena.tasks.schema.elo import EloProtocol from judgearena.tasks.schema.mt_bench import MTBenchProtocol from judgearena.tasks.schema.pairwise import PairwiseProtocol ProtocolSpec = Annotated[ - PairwiseProtocol | MTBenchProtocol, + PairwiseProtocol | MTBenchProtocol | EloProtocol, Field(discriminator="runner"), ] diff --git a/judgearena/utils/io.py b/judgearena/utils/io.py index 9e5c7f9..b2181d6 100644 --- a/judgearena/utils/io.py +++ b/judgearena/utils/io.py @@ -34,9 +34,9 @@ def download_hf(name: str, local_path: Path): resolved_task = get_packaged_task(name) if resolved_task is not None: - from judgearena.datasets.registry import resolve_dataset_adapter + from judgearena.datasets.registry import resolve_download_adapter - resolve_dataset_adapter(resolved_task.spec.dataset.adapter).download( + resolve_download_adapter(resolved_task.spec.dataset.adapter).download( resolved_task, local_path ) else: diff --git a/tests/test_cli.py b/tests/test_cli.py index 4fc41e6..b0264dd 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -10,20 +10,15 @@ @pytest.fixture def capture_mains(monkeypatch): - """Replace both main functions (and logging) with spies recording the config.""" + """Replace benchmark execution and logging with a config spy.""" captured: dict[str, object] = {} - def fake_main_ge(cfg: RunConfig) -> None: - captured["module"] = "generate_and_evaluate" - captured["cfg"] = cfg - - def fake_main_elo(cfg: RunConfig) -> None: - captured["module"] = "elo" + def fake_run_benchmark(cfg: RunConfig) -> None: + captured["module"] = "benchmark" captured["cfg"] = cfg monkeypatch.setattr(cli_module, "configure_logging", lambda *a, **k: None) - monkeypatch.setattr(cli_module, "run_benchmark", fake_main_ge) - monkeypatch.setattr(cli_module, "main_elo", fake_main_elo) + monkeypatch.setattr(cli_module, "run_benchmark", fake_run_benchmark) return captured @@ -37,7 +32,7 @@ def fake_main_elo(cfg: RunConfig) -> None: "mt-bench", ], ) -def test_task_dispatches_to_generate_and_evaluate(capture_mains, task: str): +def test_task_dispatches_to_benchmark_registry(capture_mains, task: str): cli_module.cli( [ "--task", @@ -50,7 +45,7 @@ def test_task_dispatches_to_generate_and_evaluate(capture_mains, task: str): "Dummy/J", ] ) - assert capture_mains["module"] == "generate_and_evaluate" + assert capture_mains["module"] == "benchmark" cfg = capture_mains["cfg"] assert isinstance(cfg, RunConfig) assert cfg.task == task @@ -72,7 +67,7 @@ def test_elo_task_dispatches(capture_mains, task: str, expected_arena: str): cli_module.cli( ["--task", task, "--model.name", "Dummy/X", "--judge.model", "Dummy/J"] ) - assert capture_mains["module"] == "elo" + assert capture_mains["module"] == "benchmark" cfg = capture_mains["cfg"] assert isinstance(cfg, RunConfig) assert cfg.elo is not None @@ -113,12 +108,12 @@ def test_missing_judge_errors(capture_mains): def test_elo_requires_model_path(capture_mains): - with pytest.raises(SystemExit, match="model.name is required for elo"): + with pytest.raises(SystemExit, match="model.name is required for ELO"): cli_module.cli(["--task", "elo-comparia", "--judge.model", "Dummy/J"]) def test_elo_rejects_model_path_b(capture_mains): - with pytest.raises(SystemExit, match="model.baseline is not supported for elo"): + with pytest.raises(SystemExit, match="model.baseline is not supported for ELO"): cli_module.cli( [ "--task", @@ -134,7 +129,7 @@ def test_elo_rejects_model_path_b(capture_mains): def test_unknown_elo_task_errors(capture_mains): - with pytest.raises(SystemExit, match="Unknown elo task"): + with pytest.raises(SystemExit, match="Unknown task"): cli_module.cli( ["--task", "elo-foo", "--model.name", "Dummy/X", "--judge.model", "Dummy/J"] ) diff --git a/tests/test_config.py b/tests/test_config.py index f53ded7..8b24e53 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -95,6 +95,27 @@ def test_elo_config_derives_arena(): cfg = RunConfig(**_base_elo()) assert cfg.elo is not None assert cfg.elo.arena == "ComparIA" + assert cfg.elo.soft_elo is True + assert cfg.elo.soft_elo_temperature == 0.3 + + +def test_elo_config_rejects_arena_that_conflicts_with_task(): + data = _base_elo() + data["elo"] = {"arena": "LMArena-100k"} + + with pytest.raises(ValidationError, match="does not match task"): + RunConfig(**data) + + +def test_elo_config_allows_runtime_scoring_overrides(): + data = _base_elo() + data["elo"] = {"soft_elo": False, "soft_elo_temperature": 0.7} + + cfg = RunConfig(**data) + + assert cfg.elo is not None + assert cfg.elo.soft_elo is False + assert cfg.elo.soft_elo_temperature == 0.7 def test_elo_requires_model_path(): @@ -205,21 +226,19 @@ def test_cli_yaml_equivalence_elo(tmp_path): def test_config_path_dispatches_elo(tmp_path, monkeypatch): captured = {} monkeypatch.setattr(cli_module, "configure_logging", lambda *a, **k: None) - monkeypatch.setattr(cli_module, "main_elo", lambda a: captured.setdefault("elo", a)) monkeypatch.setattr( cli_module, "run_benchmark", - lambda a: captured.setdefault("ge", a), + lambda a: captured.setdefault("benchmark", a), ) yaml_path = tmp_path / "e.yaml" yaml_path.write_text( "task: elo-comparia\nmodel: {name: Dummy/m}\njudge: {model: Dummy/j}\n" ) cli_module.cli(["--config_path", str(yaml_path)]) - assert "ge" not in captured - assert isinstance(captured["elo"], RunConfig) - assert captured["elo"].elo is not None - assert captured["elo"].elo.arena == "ComparIA" + assert isinstance(captured["benchmark"], RunConfig) + assert captured["benchmark"].elo is not None + assert captured["benchmark"].elo.arena == "ComparIA" def test_build_run_config_cli_only(): diff --git a/tests/test_elo_task_runtime.py b/tests/test_elo_task_runtime.py new file mode 100644 index 0000000..aa4e403 --- /dev/null +++ b/tests/test_elo_task_runtime.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import pandas as pd + +import judgearena.datasets.arena_battles as arena_battles +from judgearena.benchmarks.elo.rating import fit_bradley_terry +from judgearena.benchmarks.elo.scoring import ELO_SCORERS +from judgearena.tasks.registry import get_packaged_task + + +def _elo_task(task_id: str = "elo-comparia"): + task = get_packaged_task(task_id) + assert task is not None + return task + + +def test_elo_dataset_adapter_uses_task_owned_arena_and_sources(monkeypatch, tmp_path): + task = _elo_task() + captured = {} + + def fake_load_arena_dataframe(arena, *, dataset_sources): + captured["arena"] = arena + captured["sources"] = dataset_sources + return pd.DataFrame([{"question_id": "q1"}]) + + monkeypatch.setattr( + arena_battles, "load_arena_dataframe", fake_load_arena_dataframe + ) + + battles = arena_battles.load_task_battles(task, tmp_path) + + source = captured["sources"]["ministere-culture/comparia-votes"] + assert captured["arena"] == "ComparIA" + assert source.revision == "7a40bce496c1f2aa3be4001da85a49cb4743042b" + assert battles["question_id"].tolist() == ["q1"] + + +def test_elo_dataset_download_uses_pinned_task_source(monkeypatch, tmp_path): + task = _elo_task("elo-lmarena-100k") + captured = {} + monkeypatch.setattr( + arena_battles, + "snapshot_download", + lambda **kwargs: captured.update(kwargs), + ) + + arena_battles.download_task_sources(task, tmp_path) + + assert captured["repo_id"] == "lmarena-ai/arena-human-preference-100k" + assert captured["revision"] == "72e85b3ddc9c81bf7b659d6b03d4126dfd8fb34a" + assert captured["allow_patterns"] == ("data/*.parquet",) + + +def test_elo_scoring_adapter_resolves_task_selection(): + task = _elo_task() + + assert ELO_SCORERS[task.spec.protocol.scoring.adapter].fit is fit_bradley_terry diff --git a/tests/test_estimate_elo_ratings.py b/tests/test_estimate_elo_ratings.py index 75424b2..41b4bdc 100644 --- a/tests/test_estimate_elo_ratings.py +++ b/tests/test_estimate_elo_ratings.py @@ -10,10 +10,11 @@ fit_bradley_terry, winner_to_pref, ) -from judgearena.benchmarks.elo.runner import main +from judgearena.benchmarks.elo.runner import run_elo from judgearena.config import RunConfig from judgearena.evaluate import JudgeAnnotation, judge_and_parse_prefs from judgearena.models import make_model +from judgearena.tasks.registry import get_packaged_task N_BATTLES = 30 ARENA_MODELS = ["arena_model_alpha", "arena_model_beta", "arena_model_gamma"] @@ -26,12 +27,11 @@ def _make_conversation(content_user: str, content_assistant: str) -> list[dict]: ] -@pytest.fixture -def synthetic_arena_df() -> pd.DataFrame: +def _arena_df(n_battles: int) -> pd.DataFrame: """Synthetic arena DataFrame matching the schema produced by load_arena_dataframe.""" rng = np.random.default_rng(42) rows = [] - for i in range(N_BATTLES): + for i in range(n_battles): ma, mb = rng.choice(ARENA_MODELS, size=2, replace=False) winner = rng.choice(["model_a", "model_b", "tie"]) lang = rng.choice(["en", "fr"]) @@ -55,12 +55,17 @@ def synthetic_arena_df() -> pd.DataFrame: return pd.DataFrame(rows) +@pytest.fixture +def synthetic_arena_df() -> pd.DataFrame: + return _arena_df(N_BATTLES) + + @pytest.fixture(autouse=True) def mock_external_deps(monkeypatch, synthetic_arena_df): monkeypatch.setattr( estimate_elo_ratings, - "load_arena_dataframe", - lambda arena: synthetic_arena_df, + "load_battles", + lambda _task: synthetic_arena_df, ) def mock_generate(instructions, model, **kwargs): @@ -84,7 +89,8 @@ def _run_without_cache(fun, **_kwargs): def _default_args(*, result_folder: str, **kwargs) -> RunConfig: - arena = kwargs.pop("arena", "ComparIA") + task = kwargs.pop("task", "elo-comparia") + arena = kwargs.pop("arena", None) model = kwargs.pop("model", "Dummy/my model") judge_model = kwargs.pop("judge_model", "Dummy/score A: 0 score B: 10") n_instructions = kwargs.pop("n_instructions", 10) @@ -92,6 +98,7 @@ def _default_args(*, result_folder: str, **kwargs) -> RunConfig: languages = kwargs.pop("languages", None) swap_mode = kwargs.pop("swap_mode", "fixed") strip_thinking_before_judging = kwargs.pop("strip_thinking_before_judging", False) + calibrate_temperature = kwargs.pop("calibrate_temperature", False) battle_thinking_token_budget = kwargs.pop("battle_thinking_token_budget", None) assert not kwargs, f"unexpected kwargs: {kwargs}" judge: dict[str, object] = { @@ -102,11 +109,16 @@ def _default_args(*, result_folder: str, **kwargs) -> RunConfig: if battle_thinking_token_budget is not None: judge["battle_thinking_token_budget"] = battle_thinking_token_budget return RunConfig( - task="elo-comparia", + task=task, model={"name": model}, judge=judge, generation={"n_instructions": n_instructions}, - elo={"arena": arena, "n_bootstraps": n_bootstraps, "languages": languages}, + elo={ + "arena": arena, + "n_bootstraps": n_bootstraps, + "languages": languages, + "calibrate_temperature": calibrate_temperature, + }, run={"result_folder": result_folder}, ) @@ -164,11 +176,15 @@ def test_bradley_terry_soft_matches_hard(): assert hard["B"] == pytest.approx(soft["B"], abs=1e-3) -# --- main() integration tests --- +# --- run_elo() integration tests --- + + +def run_elo_with_task(cfg: RunConfig) -> dict: + return run_elo(cfg, get_packaged_task(cfg.task)) -def test_main_returns_summary(tmp_path): - result = main(_default_args(result_folder=str(tmp_path))) +def test_run_elo_returns_summary(tmp_path): + result = run_elo_with_task(_default_args(result_folder=str(tmp_path))) assert set(result.keys()) >= { "num_wins", "num_losses", @@ -179,21 +195,21 @@ def test_main_returns_summary(tmp_path): } -def test_main_winrate_in_valid_range(tmp_path): - result = main(_default_args(result_folder=str(tmp_path))) +def test_run_elo_winrate_in_valid_range(tmp_path): + result = run_elo_with_task(_default_args(result_folder=str(tmp_path))) assert 0.0 <= result["winrate"] <= 1.0 -def test_main_winrate_depends_on_judge(tmp_path): +def test_run_elo_winrate_depends_on_judge(tmp_path): """A judge biased toward one position should yield different winrates depending on direction.""" # With seed=0 and n=10 our model is always placed in position B, so: # judge favouring B → all wins; judge favouring A → all losses - result_wins = main( + result_wins = run_elo_with_task( _default_args( result_folder=str(tmp_path), judge_model="Dummy/score A: 0 score B: 10" ) ) - result_loses = main( + result_loses = run_elo_with_task( _default_args( result_folder=str(tmp_path), judge_model="Dummy/score A: 10 score B: 0" ) @@ -201,10 +217,12 @@ def test_main_winrate_depends_on_judge(tmp_path): assert result_wins["winrate"] > result_loses["winrate"] -def test_main_language_filter_reduces_battles(tmp_path): +def test_run_elo_language_filter_reduces_battles(tmp_path): """Filtering to a single language should use fewer battles than no filter.""" - result_all = main(_default_args(result_folder=str(tmp_path), n_instructions=None)) - result_en = main( + result_all = run_elo_with_task( + _default_args(result_folder=str(tmp_path), n_instructions=None) + ) + result_en = run_elo_with_task( _default_args( result_folder=str(tmp_path), n_instructions=None, languages=["en"] ) @@ -216,17 +234,21 @@ def test_main_language_filter_reduces_battles(tmp_path): assert total_en < total_all -def test_main_model_in_bootstrap_ratings(tmp_path): +def test_run_elo_model_in_bootstrap_ratings(tmp_path): """Our model should appear in the bootstrap ELO leaderboard.""" - result = main(_default_args(result_folder=str(tmp_path))) + result = run_elo_with_task(_default_args(result_folder=str(tmp_path))) model_name = result["model_name"] assert all(model_name in r for r in result["bootstrap_ratings"]) -def test_main_n_instructions_limits_battles(tmp_path): +def test_run_elo_n_instructions_limits_battles(tmp_path): """n_instructions caps the number of judged battles.""" - result_5 = main(_default_args(result_folder=str(tmp_path), n_instructions=5)) - result_10 = main(_default_args(result_folder=str(tmp_path), n_instructions=10)) + result_5 = run_elo_with_task( + _default_args(result_folder=str(tmp_path), n_instructions=5) + ) + result_10 = run_elo_with_task( + _default_args(result_folder=str(tmp_path), n_instructions=10) + ) total_5 = ( result_5["num_wins"] + result_5["num_losses"] @@ -243,7 +265,7 @@ def test_main_n_instructions_limits_battles(tmp_path): assert total_10 == 10 -def test_main_swap_mode_forwarded_to_judge(monkeypatch, tmp_path): +def test_run_elo_swap_mode_forwarded_to_judge(monkeypatch, tmp_path): """swap_mode from the run config must be forwarded to judge_and_parse_prefs. Regression test: previously run_judge() called judge_and_parse_prefs without @@ -270,7 +292,7 @@ def spy_judge( return [dummy] * n, None, pd.Series([1.0] * n) monkeypatch.setattr(estimate_elo_ratings, "judge_and_parse_prefs", spy_judge) - main(_default_args(result_folder=str(tmp_path), swap_mode="both")) + run_elo_with_task(_default_args(result_folder=str(tmp_path), swap_mode="both")) assert captured.get("swap_mode") == "both" @@ -297,7 +319,7 @@ def spy_judge( return spy_judge -def test_main_strip_thinking_forwarded_to_judge(monkeypatch, tmp_path): +def test_run_elo_strip_thinking_forwarded_to_judge(monkeypatch, tmp_path): """strip_thinking_before_judging from the run config must reach the judge. Regression test: the Elo entrypoint accepted the flag but never forwarded it @@ -307,16 +329,18 @@ def test_main_strip_thinking_forwarded_to_judge(monkeypatch, tmp_path): monkeypatch.setattr( estimate_elo_ratings, "judge_and_parse_prefs", _spy_judge_capturing(captured) ) - main(_default_args(result_folder=str(tmp_path), strip_thinking_before_judging=True)) + run_elo_with_task( + _default_args(result_folder=str(tmp_path), strip_thinking_before_judging=True) + ) assert captured.get("strip_thinking_before_judging") is True -def test_main_strip_thinking_defaults_off(monkeypatch, tmp_path): +def test_run_elo_strip_thinking_defaults_off(monkeypatch, tmp_path): captured = {} monkeypatch.setattr( estimate_elo_ratings, "judge_and_parse_prefs", _spy_judge_capturing(captured) ) - main(_default_args(result_folder=str(tmp_path))) + run_elo_with_task(_default_args(result_folder=str(tmp_path))) assert captured.get("strip_thinking_before_judging") is False @@ -333,13 +357,13 @@ def spy_generate(instructions, model, **kwargs): return spy_generate -def test_main_thinking_budget_injected_for_thinking_model(monkeypatch, tmp_path): +def test_run_elo_thinking_budget_injected_for_thinking_model(monkeypatch, tmp_path): """battle_thinking_token_budget must reach generation for VLLM thinking models.""" captured = {} monkeypatch.setattr( estimate_elo_ratings, "generate_instructions", _spy_generate_capturing(captured) ) - main( + run_elo_with_task( _default_args( result_folder=str(tmp_path), model="VLLM/Qwen/Qwen3.5-9B", @@ -349,7 +373,7 @@ def test_main_thinking_budget_injected_for_thinking_model(monkeypatch, tmp_path) assert captured["gen_kwargs"].get("thinking_token_budget") == 128 -def test_main_thinking_budget_capped_by_max_out_tokens(monkeypatch, tmp_path): +def test_run_elo_thinking_budget_capped_by_max_out_tokens(monkeypatch, tmp_path): captured = {} monkeypatch.setattr( estimate_elo_ratings, "generate_instructions", _spy_generate_capturing(captured) @@ -359,18 +383,18 @@ def test_main_thinking_budget_capped_by_max_out_tokens(monkeypatch, tmp_path): model="VLLM/Qwen/Qwen3.5-9B", battle_thinking_token_budget=10**9, ) - main(cfg) + run_elo_with_task(cfg) assert ( captured["gen_kwargs"].get("thinking_token_budget") == cfg.model.max_out_tokens ) -def test_main_thinking_budget_absent_for_nonthinking_model(monkeypatch, tmp_path): +def test_run_elo_thinking_budget_absent_for_nonthinking_model(monkeypatch, tmp_path): captured = {} monkeypatch.setattr( estimate_elo_ratings, "generate_instructions", _spy_generate_capturing(captured) ) - main( + run_elo_with_task( _default_args( result_folder=str(tmp_path), model="Dummy/my model", @@ -423,3 +447,65 @@ def test_arena_anchor_battles_filters_and_preserves_index(): assert list(out.index) == list(range(1000, 1000 + n)) # labels preserved, rare gone assert (out["source"] == "human").all() assert out.loc[1000, "pref"] == 0.0 and out.loc[1001, "pref"] == 1.0 + + +def test_elo_language_variant_resolves_and_filters(tmp_path): + variant = get_packaged_task("elo-lmarena-140k-en") + assert variant is not None + assert variant.selection is not None + assert variant.selection.values == ("en",) + + result_en = run_elo( + _default_args( + result_folder=str(tmp_path), task="elo-lmarena-140k-en", n_instructions=None + ), + variant, + ) + result_all = run_elo_with_task( + _default_args( + result_folder=str(tmp_path), task="elo-lmarena-140k", n_instructions=None + ) + ) + total_en = result_en["num_wins"] + result_en["num_losses"] + result_en["num_ties"] + total_all = ( + result_all["num_wins"] + result_all["num_losses"] + result_all["num_ties"] + ) + assert 0 < total_en < total_all + + +def test_run_elo_temperature_calibration_builds_judge(monkeypatch, tmp_path): + """Regression: the calibration path constructs its own judge model and once + crashed on a duplicate max_tokens kwarg; nothing else exercises it. The + MLE fit itself is mocked.""" + captured = {} + + def fake_calibrate(delta_s, y): + captured["n_pairs"] = len(delta_s) + return 0.42 + + monkeypatch.setattr(estimate_elo_ratings, "calibrate_temperature", fake_calibrate) + # Anchor battles require models with >= 500 appearances; the default + # 30-battle fixture leaves the calibration pool empty. + monkeypatch.setattr( + estimate_elo_ratings, "load_battles", lambda _task: _arena_df(900) + ) + + result = run_elo_with_task( + _default_args(result_folder=str(tmp_path), calibrate_temperature=True) + ) + + assert captured["n_pairs"] >= 10 + assert 0.0 <= result["winrate"] <= 1.0 + + +def test_extract_instruction_text_tolerates_moderated_turns(): + from judgearena.arenas_utils import _extract_instruction_text + + assert _extract_instruction_text({"content": None}) == "" + assert _extract_instruction_text({"content": "plain"}) == "plain" + assert ( + _extract_instruction_text( + {"content": [{"type": "text", "text": None}, {"type": "image"}, None]} + ) + == "" + ) diff --git a/tests/test_generate_and_evaluate.py b/tests/test_generate_and_evaluate.py index fbd6c4e..2313abd 100644 --- a/tests/test_generate_and_evaluate.py +++ b/tests/test_generate_and_evaluate.py @@ -179,6 +179,7 @@ def test_native_pairwise_baseline_resolves_registered_tasks(task: str, expected: [ ("alpaca-eval", "pairwise"), ("mt-bench", "mt_bench"), + ("elo-comparia", "elo"), ], ) def test_benchmark_adapter_resolution(task: str, expected: str): diff --git a/tests/test_instruction_dataset.py b/tests/test_instruction_dataset.py index 2e57a5c..1355e37 100644 --- a/tests/test_instruction_dataset.py +++ b/tests/test_instruction_dataset.py @@ -398,7 +398,6 @@ def test_pairwise_task_data_uses_declared_adapter_outputs(monkeypatch, tmp_path) ).to_csv(output_path, index=False) adapter = SimpleNamespace( - name="arena_hard", load_instructions=lambda task, path: pd.DataFrame( { "instruction_index": [0, 1], diff --git a/tests/test_mt_bench_downloads.py b/tests/test_mt_bench_downloads.py index 26f5eeb..4191d47 100644 --- a/tests/test_mt_bench_downloads.py +++ b/tests/test_mt_bench_downloads.py @@ -116,6 +116,10 @@ def _contexts_snapshot_stub(**_kwargs): "alpaca-eval", "arena-hard-v0.1", "arena-hard-v2.0", + "elo-comparia", + "elo-lmarena", + "elo-lmarena-100k", + "elo-lmarena-140k", "m-arena-hard-v0.1", "m-arena-hard-v2.0", "mt-bench", diff --git a/tests/test_task_registry.py b/tests/test_task_registry.py index 13d49e2..44db321 100644 --- a/tests/test_task_registry.py +++ b/tests/test_task_registry.py @@ -70,6 +70,8 @@ def test_packaged_registry_discovers_versioned_tasks(): arena_v01 = tasks["arena-hard-v0.1"] arena_v20 = tasks["arena-hard-v2.0"] + elo_comparia = tasks["elo-comparia"] + elo_lmarena = tasks["elo-lmarena"] m_arena_v01 = tasks["m-arena-hard-v0.1"] m_arena_eu = resolve_task(tasks, "m-arena-hard-v2.0-EU") assert m_arena_eu is not None @@ -79,6 +81,10 @@ def test_packaged_registry_discovers_versioned_tasks(): "alpaca-eval", "arena-hard-v0.1", "arena-hard-v2.0", + "elo-comparia", + "elo-lmarena", + "elo-lmarena-100k", + "elo-lmarena-140k", "m-arena-hard-v0.1", "m-arena-hard-v2.0", "mt-bench", @@ -91,6 +97,14 @@ def test_packaged_registry_discovers_versioned_tasks(): assert arena_v20.spec.protocol.baseline.references["hard_prompt"] == ( "o3-mini-2025-01-31" ) + assert elo_comparia.spec.protocol.runner == "elo" + assert elo_comparia.spec.protocol.arena == "ComparIA" + assert elo_comparia.spec.protocol.scoring.adapter == "bradley_terry" + assert elo_comparia.spec.dataset.sources["comparia"].revision == ( + "7a40bce496c1f2aa3be4001da85a49cb4743042b" + ) + assert elo_lmarena.spec.protocol.arena == "LMArena" + assert len(elo_lmarena.spec.dataset.sources) == 3 assert [resource.path for resource in arena_v20.provenance.resources] == [ "arena_hard/_base.yaml", "arena_hard/arena-hard-v2.0.yaml", @@ -334,6 +348,20 @@ def test_registry_rejects_unknown_adapter_id(tmp_path): load_tasks(tmp_path) +def test_registry_rejects_dataset_adapter_from_another_protocol(tmp_path): + definition = _task_definition() + definition["dataset"]["adapter"] = "arena_battles" + _write_family( + tmp_path, + family="example", + filename="test-task.yaml", + definition=definition, + ) + + with pytest.raises(TaskDefinitionError, match="unknown dataset adapter"): + load_tasks(tmp_path) + + def test_registry_rejects_unknown_scorer_id(tmp_path): definition = _task_definition() definition["protocol"]["scoring"]["adapter"] = "missing_scorer" @@ -348,6 +376,20 @@ def test_registry_rejects_unknown_scorer_id(tmp_path): load_tasks(tmp_path) +def test_registry_rejects_scorer_from_another_protocol(tmp_path): + definition = _task_definition() + definition["protocol"]["scoring"]["adapter"] = "bradley_terry" + _write_family( + tmp_path, + family="example", + filename="test-task.yaml", + definition=definition, + ) + + with pytest.raises(TaskDefinitionError, match="unknown scorer"): + load_tasks(tmp_path) + + def test_official_outputs_must_reference_declared_source(tmp_path): definition = _task_definition() definition["protocol"]["baseline"] = { diff --git a/tests/test_utils.py b/tests/test_utils.py index 2d5f26f..dd305ef 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -49,15 +49,19 @@ def test_download_all_dispatches_registered_tasks(monkeypatch, tmp_path): utils_io.download_all() tables_dir = tmp_path / "tables" - assert calls[:6] == [ + assert calls[:10] == [ ("hf", "alpaca-eval", tables_dir), ("hf", "arena-hard-v0.1", tables_dir), ("hf", "arena-hard-v2.0", tables_dir), + ("hf", "elo-comparia", tables_dir), + ("hf", "elo-lmarena", tables_dir), + ("hf", "elo-lmarena-100k", tables_dir), + ("hf", "elo-lmarena-140k", tables_dir), ("hf", "m-arena-hard-v0.1", tables_dir), ("hf", "m-arena-hard-v2.0", tables_dir), ("hf", "mt-bench", tables_dir), ] - assert calls[6] == ( + assert calls[10] == ( "snapshot", "geoalgo/multilingual-fluency", tmp_path / "multilingual-fluency",