(8/n) Add ELO/Battles datasets as Task

README.md

@@ -240,7 +240,7 @@ This override applies to all vLLM models in the run. For remote providers (OpenA
 
 ## 📊 Supported Tasks
 
-Task names follow [LMHarness](https://github.com/EleutherAI/lm-evaluation-harness) conventions. Generate+judge tasks produce pairwise preferences between two models; ELO tasks (`elo-*`) estimate a single model's ELO rating against human-annotated arena opponents.
+Task names follow [LMHarness](https://github.com/EleutherAI/lm-evaluation-harness) conventions. Generate+judge tasks produce pairwise preferences between two models; tasks using the ELO protocol estimate a single model's rating against human-annotated arena opponents.
 
 ### Generate + judge (pairwise)
 
@@ -289,7 +289,7 @@ For m-Arena-Hard, baseline completions are tied to the benchmark release:
 JudgeArena can estimate the ELO rating of a model by running it against opponents sampled from a human preference arena (`LMArena-100k`, `LMArena-140k`, or `ComparIA`).
 The LLM judge scores each battle, and the resulting ratings are computed using the Bradley-Terry model anchored against the human-annotated arena leaderboard.
 
-Pass an `elo-<arena>` value to `--task` to trigger the ELO flow. ELO tasks take a single `--model.name` whose opponents are sampled from the arena (matching the pairwise CLI shape; `--model.baseline` is reserved for a future extension).
+Select one of the packaged ELO tasks with `--task`. Its task definition selects the ELO runner, canonical arena, pinned battle sources, judge defaults, and Bradley-Terry scoring adapter. ELO tasks take a single `--model.name`; experiment settings such as sampling, filtering, bootstrapping, and calibration remain runtime options. `--model.baseline` is not used because opponents come from the selected arena.
 
 ### Quick start
 
@@ -305,7 +305,7 @@ judgearena \
 
 | Flag | Default | Description |
 |---|---|---|
-| `--task elo-<arena>` | *(required)* | Arena to sample opponents from: `elo-lmarena-100k`, `elo-lmarena-140k`, `elo-lmarena`, or `elo-comparia` |
+| `--task` | *(required)* | Packaged ELO task: `elo-lmarena-100k`, `elo-lmarena-140k`, `elo-lmarena`, or `elo-comparia` |
 | `--model.name` | *(required)* | Model under evaluation (same format as pairwise tasks) |
 | `--judge.model` | *(required)* | LLM judge (same format as pairwise tasks) |
 | `--generation.n_instructions` | all | Number of arena battles to use for evaluation |

configs/elo_comparia.yaml

@@ -1,7 +1,7 @@
 # Example JudgeArena ELO-rating run config.
 # Run with:  judgearena --config_path configs/elo_comparia.yaml
 # ELO tasks rate a single model (model.name) against arena opponents; the
-# arena is derived from the elo-* task, so elo.arena does not need to be set.
+# arena and pinned battle sources come from the packaged task definition.
 task: elo-comparia
 model:
   name: Together/meta-llama/Llama-3.3-70B-Instruct-Turbo

judgearena/arenas_utils.py

@@ -1,43 +1,73 @@
 import warnings
+from collections.abc import Mapping
 from pathlib import Path
 
 import pandas as pd
 from fast_langdetect import detect_language
 from huggingface_hub import snapshot_download
 
-from judgearena.dataset_revisions import hf_revision
 from judgearena.log import get_logger
+from judgearena.tasks.schema import HuggingFaceDatasetSource
 
 logger = get_logger(__name__)
 
 
+def _download_arena_dataset(
+    *,
+    repo_id: str,
+    default_allow_patterns: str | tuple[str, ...],
+    dataset_sources: Mapping[str, HuggingFaceDatasetSource],
+) -> str:
+    """Download one arena source at the revision pinned by its task definition."""
+    try:
+        source = dataset_sources[repo_id]
+    except KeyError as exc:
+        raise ValueError(
+            f"Arena task does not declare required dataset source {repo_id!r}."
+        ) from exc
+    return snapshot_download(
+        repo_id=repo_id,
+        repo_type="dataset",
+        allow_patterns=source.allow_patterns or default_allow_patterns,
+        force_download=False,
+        revision=source.revision,
+    )
+
+
 def _extract_instruction_text(turn: dict) -> str:
-    """Extract plain instruction text from a conversation first turn.
+    """Extract plain instruction text from a conversation turn.
 
     Handles both the 100k schema (content is a plain string) and the 140k
-    schema (content is an array of {type, text, ...} objects).
+    schema (content is an array of {type, text, ...} objects). Moderated or
+    empty turns ship ``content: None`` and yield an empty string.
     """
-    content = turn["content"]
+    content = turn.get("content")
+    if content is None:
+        return ""
     if isinstance(content, str):
         return content
-    return " ".join(block["text"] for block in content if block.get("type") == "text")
+    return " ".join(
+        block.get("text") or ""
+        for block in content
+        if isinstance(block, dict) and block.get("type") == "text"
+    )
 
 
 KNOWN_ARENAS = ["LMArena-100k", "LMArena-55k", "LMArena-140k", "ComparIA"]
 
 
 def _load_arena_dataframe(
-    arena: str, comparia_revision: str | None = None
+    arena: str,
+    *,
+    dataset_sources: Mapping[str, HuggingFaceDatasetSource],
 ) -> pd.DataFrame:
     assert arena in KNOWN_ARENAS
     if arena == "LMArena-55k":
         repo_id = "lmarena-ai/arena-human-preference-55k"
-        path = snapshot_download(
+        path = _download_arena_dataset(
             repo_id=repo_id,
-            repo_type="dataset",
-            allow_patterns="*.csv",
-            force_download=False,
-            revision=hf_revision(repo_id),
+            default_allow_patterns="*.csv",
+            dataset_sources=dataset_sources,
         )
         df = pd.read_csv(Path(path) / "train.csv")
 
@@ -74,12 +104,10 @@ def _winner_55k(row) -> str | None:
     elif "LMArena" in arena:
         size = arena.split("-")[1]  # "100k" or "140k"
         repo_id = f"lmarena-ai/arena-human-preference-{size}"
-        path = snapshot_download(
+        path = _download_arena_dataset(
             repo_id=repo_id,
-            repo_type="dataset",
-            allow_patterns="*parquet",
-            force_download=False,
-            revision=hf_revision(repo_id),
+            default_allow_patterns="*parquet",
+            dataset_sources=dataset_sources,
         )
         parquet_files = sorted((Path(path) / "data").glob("*.parquet"))
         df = pd.concat([pd.read_parquet(f) for f in parquet_files], ignore_index=True)
@@ -101,12 +129,10 @@ def _winner_55k(row) -> str | None:
         df["benchmark"] = arena
 
     else:
-        path = snapshot_download(
+        path = _download_arena_dataset(
             repo_id="ministere-culture/comparia-votes",
-            repo_type="dataset",
-            allow_patterns="*",
-            revision=comparia_revision,
-            force_download=False,
+            default_allow_patterns="*",
+            dataset_sources=dataset_sources,
         )
 
         df = pd.read_parquet(Path(path) / "votes.parquet")
@@ -176,36 +202,41 @@ def get_winner(
     return df
 
 
-_DEFAULT_COMPARIA_REVISION = hf_revision("ministere-culture/comparia-votes")
-
-
 def load_arena_dataframe(
     arena: str | None,
-    comparia_revision: str | None = _DEFAULT_COMPARIA_REVISION,
+    *,
+    dataset_sources: Mapping[str, HuggingFaceDatasetSource],
 ) -> pd.DataFrame:
     """Load battles from one or all arenas.
 
     :param arena: one of "LMArena-100k", "LMArena-140k", "ComparIA", "LMArena"
                   (concatenation of both LMArena variants), or None (all arenas).
-    :param comparia_revision: pinned revision for the ComparIA dataset.
+    :param dataset_sources: pinned sources declared by the task, keyed by repo ID.
     :return: dataframe containing battles for the arena(s) selected.
     """
     if arena is None:
         arenas = KNOWN_ARENAS
     elif arena == "LMArena":
         arenas = ["LMArena-100k", "LMArena-55k", "LMArena-140k"]
     else:
-        return _load_arena_dataframe(arena, comparia_revision)
+        return _load_arena_dataframe(arena, dataset_sources=dataset_sources)
     return pd.concat(
-        [_load_arena_dataframe(a, comparia_revision) for a in arenas],
+        [_load_arena_dataframe(a, dataset_sources=dataset_sources) for a in arenas],
         ignore_index=True,
     )
 
 
 def main():
-    for arena in KNOWN_ARENAS:
-        logger.info("Loading %s", arena)
-        df = _load_arena_dataframe(arena)
+    from judgearena.datasets import load_battles
+    from judgearena.tasks.registry import load_tasks
+    from judgearena.tasks.schema import EloProtocol
+
+    for task_id, task in load_tasks().items():
+        if not isinstance(task.spec.protocol, EloProtocol):
+            continue
+        logger.info("Loading %s", task_id)
+        df = load_battles(task)
+        arena = task.spec.protocol.arena
         n_battles = len(df)
         n_models = len(set(df["model_a"]) | set(df["model_b"]))
         n_languages = df["lang"].nunique()

judgearena/benchmarks/elo/runner.py

@@ -7,7 +7,7 @@
 import numpy as np
 import pandas as pd
 
-from judgearena.arenas_utils import _extract_instruction_text, load_arena_dataframe
+from judgearena.arenas_utils import _extract_instruction_text
 from judgearena.artifacts import (
     prepare_run_directory,
     safe_filename,
@@ -16,13 +16,14 @@
 from judgearena.battles import Leaderboard, summarize_bootstrap, write_battles
 from judgearena.benchmarks.elo.rating import (
     arena_anchor_battles,
-    fit_bradley_terry,
     prefs_to_battle_results,
     sampling_cache_token,
     select_seeded_random_arena_battles,
     winner_to_pref,
 )
+from judgearena.benchmarks.elo.scoring import ELO_SCORERS
 from judgearena.benchmarks.execution import build_generation_kwargs
+from judgearena.datasets import load_battles
 from judgearena.evaluate import (
     PairScore,
     calibrate_temperature,
@@ -33,6 +34,7 @@
 from judgearena.generate import generate_instructions
 from judgearena.log import get_logger
 from judgearena.models import build_default_judge_model_kwargs, make_model
+from judgearena.tasks.schema import EloProtocol, ResolvedTaskSpec
 from judgearena.utils import cache_function_dataframe, compute_pref_summary
 from judgearena.utils.eval import PrefSummary, Report
 
@@ -135,19 +137,42 @@ def render(self) -> None:
             print("\n  No overlapping arena models to compute MAE.")
 
 
-def main(cfg: "RunConfig") -> dict:
-    assert cfg.elo is not None  # main is dispatched only for elo tasks
+def run_elo(cfg: "RunConfig", task: ResolvedTaskSpec | None = None) -> dict:
+    """Rate one model against the human battles defined by an ELO task."""
+    protocol = task.spec.protocol if task is not None else None
+    if not isinstance(protocol, EloProtocol):
+        raise ValueError(f"Task {cfg.task!r} does not define an ELO protocol.")
+    if cfg.elo is None:
+        raise ValueError(f"Task {cfg.task!r} requires ELO runtime settings.")
+    arena = protocol.arena
+    scorer = ELO_SCORERS[protocol.scoring.adapter]
     run_started_at = datetime.now(UTC)
     rng = np.random.default_rng(cfg.run.seed)
 
     # Step 1: Load arena battles
-    logger.info("Step 1: Loading battles from %s", cfg.elo.arena)
-    df_arena_all = load_arena_dataframe(arena=cfg.elo.arena)
+    logger.info("Step 1: Loading battles from %s", arena)
+    df_arena_all = load_battles(task)
+
+    # Filter by language: a task variant (e.g. elo-lmarena-140k-en) preselects
+    # languages; elo.languages narrows further within that selection.
+    selected_languages = list(cfg.elo.languages or [])
+    if task.selection is not None:
+        variant_languages = list(task.selection.values)
+        if selected_languages:
+            selected_languages = [
+                lang for lang in selected_languages if lang in set(variant_languages)
+            ]
+            if not selected_languages:
+                raise ValueError(
+                    f"elo.languages {cfg.elo.languages} has no overlap with the "
+                    f"languages of task {cfg.task!r} ({variant_languages})."
+                )
+        else:
+            selected_languages = variant_languages
 
-    # Filter by language if specified
     df_battles = df_arena_all
-    if cfg.elo.languages:
-        df_battles = df_battles[df_battles["lang"].isin(cfg.elo.languages)]
+    if selected_languages:
+        df_battles = df_battles[df_battles["lang"].isin(selected_languages)]
 
     random_sampling = cfg.elo.elo_random_battles is not None
     sampling_metadata: dict[str, object] = {"sampling_mode": "head"}
@@ -211,7 +236,9 @@ def main(cfg: "RunConfig") -> dict:
     def replace_slash(s: str) -> str:
         return s.replace("/", "_")
 
-    languages_str = "-".join(sorted(cfg.elo.languages)) if cfg.elo.languages else "all"
+    languages_str = (
+        "-".join(sorted(selected_languages)) if selected_languages else "all"
+    )
     extra_kwargs_str = (
         "_".join(f"{k}={v}" for k, v in sorted(extra_kwargs.items()))
         if extra_kwargs
@@ -223,7 +250,7 @@ def replace_slash(s: str) -> str:
         n_instructions_per_language=cfg.elo.n_instructions_per_language,
     )
     cache_suffix = (
-        f"{cfg.elo.arena}_{replace_slash(cfg.model.name)}_"
+        f"{arena}_{replace_slash(cfg.model.name)}_"
         f"{cache_token}_"
         f"{languages_str}_{cfg.generation.truncate_all_input_chars}_{extra_kwargs['max_tokens']}"
         + (f"_{extra_kwargs_str}" if extra_kwargs_str else "")
@@ -268,7 +295,7 @@ def replace_slash(s: str) -> str:
     ]
 
     our_completions = completions.tolist()
-    resolved_prompt = resolve_run_judge_prompt(cfg.elo.arena, cfg.judge)
+    resolved_prompt = resolve_run_judge_prompt(cfg.task, cfg.judge)
 
     completions_A = [
         our_completions[i] if our_model_is_position_a[i] else opponent_completions[i]
@@ -394,7 +421,7 @@ def run_judge() -> pd.DataFrame:
     df_results = pd.concat([df_llm_judge, df_arena], ignore_index=True)
 
     # Compute human-only BT ratings as ground-truth reference
-    human_elo = fit_bradley_terry(
+    human_elo = scorer.fit(
         df_arena, pref_col="pref_hard", baseline_model=cfg.elo.baseline_model
     )
 
@@ -438,7 +465,6 @@ def run_judge() -> pd.DataFrame:
 
             judge_chat_model_cal = make_model(
                 model=cfg.judge.model,
-                max_tokens=cfg.judge.max_out_tokens,
                 **judge_extra_kwargs,
             )
             cal_annotations, _, cal_prefs = judge_and_parse_prefs(
@@ -540,7 +566,7 @@ def run_judge() -> pd.DataFrame:
         df_sample = df_results.sample(
             n=len(df_results), replace=True, random_state=int(rng.integers(0, 2**31))
         )
-        ratings = fit_bradley_terry(
+        ratings = scorer.fit(
             df_sample, pref_col=pref_col, baseline_model=cfg.elo.baseline_model
         )
         bootstrap_ratings.append(ratings)
@@ -570,7 +596,7 @@ def run_judge() -> pd.DataFrame:
     )
 
     report = EloReport(
-        arena=cfg.elo.arena,
+        arena=arena,
         judge_model=cfg.judge.model,
         summary=summary,
         num_battles=n,
@@ -597,17 +623,17 @@ def run_judge() -> pd.DataFrame:
     res_dir = prepare_run_directory(
         cfg,
         Path(cfg.run.result_folder)
-        / f"elo-{safe_filename(cfg.elo.arena)}-{safe_filename(model_name)}-"
+        / f"elo-{safe_filename(arena)}-{safe_filename(model_name)}-"
         f"{safe_filename(cfg.judge.model)}",
     )
     result_path = report.save(res_dir / f"results-{safe_filename(model_name)}.json")
 
     # Persist only the run's own llm-judge battles (a few KB). The human arena
     # anchors are identical across every run, so we do not duplicate them per
-    # experiment — recompute ELO by recombining with
-    # arena_anchor_battles(load_arena_dataframe(arena)). question_id is the
-    # instruction-index join key back to the arena initial table / completion
-    # cache. battles.parquet keeps pref_hard so both hard and soft ELO recompute.
+    # experiment — recompute ELO by loading this task's pinned battles again and
+    # applying arena_anchor_battles(). question_id is the join key back to the
+    # arena table / completion cache. battles.parquet keeps pref_hard so both
+    # hard and soft ELO can be recomputed.
     battle_cols = [
         "model_a",
         "model_b",
@@ -627,7 +653,7 @@ def run_judge() -> pd.DataFrame:
             res_dir / "bootstrap_ratings.csv", index=False
         )
         Leaderboard(
-            arena=cfg.elo.arena,
+            arena=arena,
             model=model_name,
             judge_model=cfg.judge.model,
             n_bootstraps=n_bootstraps,
@@ -640,7 +666,7 @@ def run_judge() -> pd.DataFrame:
     # a metadata-write failure should not sink an already-completed run.
     write_run_metadata_safely(
         output_dir=res_dir,
-        entrypoint="judgearena.benchmarks.elo.runner.main",
+        entrypoint="judgearena.benchmarks.elo.runner.run_elo",
         run=cfg.model_dump(),
         results=results,
         input_payloads=(