(3/n) Introduce declarative YAML task definitions

judgearena/__main__.py

@@ -0,0 +1,3 @@
+from judgearena.cli import cli
+
+cli()

judgearena/artifacts/metadata.py

@@ -221,6 +221,28 @@ def _compact_results(results: dict[str, Any] | None) -> dict[str, Any]:
     return payload
 
 
+def _task_definition_metadata(run: dict[str, Any]) -> dict[str, Any] | None:
+    """Return compact provenance when ``run.task`` is a packaged task."""
+    task_id = run.get("task")
+    if not isinstance(task_id, str):
+        return None
+
+    from judgearena.tasks.registry import get_packaged_task
+
+    resolved = get_packaged_task(task_id)
+    if resolved is None:
+        return None
+    return {
+        "schema_version": resolved.spec.schema_version,
+        "task_version": resolved.spec.task_version,
+        "resolved_sha256": resolved.provenance.resolved_sha256,
+        "resources": [
+            {"path": resource.path, "sha256": resource.sha256}
+            for resource in resolved.provenance.resources
+        ],
+    }
+
+
 def write_run_metadata(
     *,
     output_dir: str | Path,
@@ -262,6 +284,10 @@ def write_run_metadata(
             start_path=Path(__file__).resolve().parent
         ),
     }
+    task_definition = _task_definition_metadata(run)
+    if task_definition is not None:
+        metadata["task_definition"] = task_definition
+
     git_hash = _get_git_hash(start_path=Path(__file__).resolve().parent)
     if git_hash:
         metadata["git_hash"] = git_hash

judgearena/benchmarks/pairwise/baselines.py

@@ -10,25 +10,31 @@
     split_m_arena_hard_dataset,
 )
 from judgearena.datasets.mt_bench import MT_BENCH_BASELINES
+from judgearena.tasks.registry import get_packaged_task
+from judgearena.tasks.schema import CategoryDefaultsBaseline, TaskDefaultBaseline
 
-ALPACA_EVAL_BASELINES: dict[str, str] = {
-    "alpaca-eval": "gpt4_1106_preview",
-}
-
-PAIRWISE_BASELINES: dict[str, str | Mapping[str, str]] = {
-    **ALPACA_EVAL_BASELINES,
+LEGACY_PAIRWISE_BASELINES: dict[str, str | Mapping[str, str]] = {
     **ARENA_HARD_BASELINES,
     **M_ARENA_HARD_BASELINES,
     **MT_BENCH_BASELINES,
 }
 
 
 def native_pairwise_baseline(task: str) -> str | Mapping[str, str] | None:
-    """Return the dataset-native pairwise baseline, if the task defines one."""
-    if task in PAIRWISE_BASELINES:
-        return PAIRWISE_BASELINES[task]
+    """Return the task-defined baseline, with fallback for unmigrated tasks."""
+    resolved = get_packaged_task(task)
+    if resolved is not None:
+        baseline = resolved.spec.protocol.baseline
+        if isinstance(baseline, TaskDefaultBaseline):
+            return baseline.reference_id
+        if isinstance(baseline, CategoryDefaultsBaseline):
+            return baseline.references
+        return None
+
+    if task in LEGACY_PAIRWISE_BASELINES:
+        return LEGACY_PAIRWISE_BASELINES[task]
     parsed_m_arena_hard = split_m_arena_hard_dataset(task)
     if parsed_m_arena_hard is not None:
         version_key, _lang_or_subset = parsed_m_arena_hard
-        return PAIRWISE_BASELINES[version_key]
+        return LEGACY_PAIRWISE_BASELINES[version_key]
     return None

judgearena/benchmarks/pairwise/runner.py

@@ -25,6 +25,7 @@
 from judgearena.evaluate import judge_and_parse_prefs, resolve_run_judge_prompt
 from judgearena.generate import generate_base, generate_instructions
 from judgearena.log import get_logger
+from judgearena.tasks.registry import get_packaged_task
 from judgearena.utils import (
     cache_function_dataframe,
     compute_pref_summary,
@@ -55,14 +56,25 @@ def try_load_dataset_completions(
     or ``None`` when no pre-existing completions are found.
     """
     local_path_tables = data_root / "tables"
-    if is_arena_hard_dataset(dataset):
+    resolved_task = get_packaged_task(dataset)
+    if resolved_task is not None:
+        from judgearena.datasets.judgearena_tables import load_task_model_outputs
+
+        df_outputs = load_task_model_outputs(resolved_task, local_path_tables)
+        if df_outputs is None:
+            return None
+    elif is_arena_hard_dataset(dataset):
         download_arena_hard(dataset=dataset, local_tables_path=local_path_tables)
+        output_path = local_path_tables / "model_outputs" / f"{dataset}.csv.zip"
+        if not output_path.exists():
+            return None
+        df_outputs = read_df(output_path)
     else:
         download_hf(name=dataset, local_path=local_path_tables)
-    output_path = local_path_tables / "model_outputs" / f"{dataset}.csv.zip"
-    if not output_path.exists():
-        return None
-    df_outputs = read_df(output_path)
+        output_path = local_path_tables / "model_outputs" / f"{dataset}.csv.zip"
+        if not output_path.exists():
+            return None
+        df_outputs = read_df(output_path)
     df_outputs.loc[:, "output"] = df_outputs.loc[:, "output"].fillna("")
     df_outputs = df_outputs.pivot_table(
         index="instruction_index", columns="model", values="output", aggfunc="last"

judgearena/benchmarks/registry.py

@@ -5,6 +5,8 @@
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Protocol
 
+from judgearena.tasks.registry import get_packaged_task
+
 if TYPE_CHECKING:
     from judgearena.config import RunConfig
 
@@ -39,8 +41,17 @@ def benchmark_adapters() -> tuple[BenchmarkAdapter, ...]:
 
 
 def resolve_benchmark_adapter(task: str) -> BenchmarkAdapter:
-    """Return the first adapter supporting ``task``."""
-    for adapter in benchmark_adapters():
+    """Resolve a YAML-selected runner, then fall back for unmigrated tasks."""
+    adapters = benchmark_adapters()
+    resolved = get_packaged_task(task)
+    if resolved is not None:
+        runner_id = resolved.spec.protocol.runner
+        for adapter in adapters:
+            if adapter.name == runner_id:
+                return adapter
+        raise ValueError(f"Task {task!r} selects unavailable runner {runner_id!r}.")
+
+    for adapter in adapters:
         if adapter.supports(task):
             return adapter
     raise ValueError(f"No generate-and-evaluate adapter supports task {task!r}.")

judgearena/cli.py

@@ -1,12 +1,13 @@
-"""Unified CLI entrypoint for judgearena.
+"""Unified CLI entrypoint for JudgeArena.
 
-Builds a ``RunConfig`` from CLI flags (derived from the config model) and/or a
-``--config_path`` YAML, then dispatches to the ELO or generate-and-judge flow
-based on ``--task`` (``elo-`` prefix runs the ELO rating flow).
+Task-management subcommands are handled before the existing model-driven run
+configuration and benchmark dispatch.
 """
 
 from __future__ import annotations
 
+import sys
+
 from pydantic import ValidationError
 
 from judgearena.benchmarks.elo.runner import main as main_elo
@@ -27,17 +28,28 @@ def _format_config_error(exc: ValidationError) -> str:
 
 
 def cli(argv: list[str] | None = None) -> None:
-    try:
-        cfg = build_run_config(argv)
-    except ValidationError as exc:
-        raise SystemExit(_format_config_error(exc)) from exc
-
-    configure_logging(cfg.run.verbosity, log_file=cfg.run.log_file)
-    logger.debug("Running with config: %s", cfg.model_dump())
-    if cfg.task.startswith(ELO_TASK_PREFIX):
-        main_elo(cfg)
+    args = list(sys.argv[1:] if argv is None else argv)
+    # `judgearena tasks {list,show,validate}` inspects packaged task definitions
+    # instead of running an evaluation, so it has its own argparse grammar (see
+    # tasks/cli.py) and must be routed before build_run_config, which only parses
+    # run/eval flags and would reject the subcommand form.
+    is_task_cli = args[:1] == ["tasks"]
+    if is_task_cli:
+        from judgearena.tasks.cli import run_task_command
+
+        run_task_command(args[1:])
     else:
-        run_benchmark(cfg)
+        try:
+            cfg = build_run_config(args)
+        except ValidationError as exc:
+            raise SystemExit(_format_config_error(exc)) from exc
+
+        configure_logging(cfg.run.verbosity, log_file=cfg.run.log_file)
+        logger.debug("Running with config: %s", cfg.model_dump())
+        if cfg.task.startswith(ELO_TASK_PREFIX):
+            main_elo(cfg)
+        else:
+            run_benchmark(cfg)
 
 
 if __name__ == "__main__":

judgearena/config.py

@@ -18,6 +18,7 @@
 
 from judgearena.benchmarks.pairwise.baselines import native_pairwise_baseline
 from judgearena.constants import ELO_TASK_PREFIX, ELO_TASK_TO_ARENA
+from judgearena.tasks.registry import get_packaged_task
 
 # Set by build_run_config() for the duration of RunConfig() construction.
 _ACTIVE_CONFIG_PATH: str | None = None
@@ -366,9 +367,8 @@ class RunConfig(BaseSettings):
     )
 
     task: str
-    """Benchmark to run. Generate+judge: ``alpaca-eval``, ``arena-hard-v2.0``,
-    ``m-arena-hard-*``, ``mt-bench``, ``fluency-*``. ELO: ``elo-lmarena-100k``,
-    ``elo-lmarena-140k``, ``elo-lmarena``, ``elo-comparia``."""
+    """Benchmark task ID. Use ``judgearena tasks list`` for packaged tasks;
+    legacy ELO task IDs use the ``elo-*`` prefix."""
 
     model: ModelArgs = Field(default_factory=ModelArgs)
     """Model(s) under evaluation and their generation settings."""
@@ -387,6 +387,33 @@ class RunConfig(BaseSettings):
 
     @model_validator(mode="after")
     def _validate(self) -> RunConfig:
+        resolved_task = get_packaged_task(self.task)
+        if resolved_task is not None:
+            task_judge = resolved_task.spec.protocol.judge
+            if "swap_mode" not in self.judge.model_fields_set:
+                self.judge.swap_mode = task_judge.default_swap_mode
+            if self.judge.swap_mode not in task_judge.allowed_swap_modes:
+                raise ValueError(
+                    f"judge.swap_mode={self.judge.swap_mode!r} is not supported "
+                    f"by task {self.task!r}; choose from "
+                    f"{list(task_judge.allowed_swap_modes)}."
+                )
+            if (
+                self.judge.temperature is None
+                and task_judge.default_temperature is not None
+            ):
+                self.judge.temperature = task_judge.default_temperature
+
+            baseline = resolved_task.spec.protocol.baseline
+            if (
+                self.model.baseline is not None
+                and getattr(baseline, "allow_runtime_override", True) is False
+            ):
+                raise ValueError(
+                    f"model.baseline cannot override the baseline defined by "
+                    f"task {self.task!r}."
+                )
+
         is_elo = self.task.startswith(ELO_TASK_PREFIX)
         if is_elo:
             if self.elo is None:

judgearena/dataset_revisions.py

@@ -25,10 +25,6 @@
     # m-ArenaHard (Cohere release)
     "CohereLabs/m-ArenaHard": "ab393a96cd0b134a1acfa96e080af31e5e73a393",
     "CohereLabs/m-ArenaHard-v2.0": "24c65eff42cec85e30dd5db99d1a702c7ebaa8ab",
-    # AlpacaEval instructions / model_outputs. Originally ``geoalgo/llmjudge``,
-    # since renamed to ``judge-arena/judge-arena-dataset`` upstream (the old id
-    # still redirects, and the commit SHA is shared across the rename).
-    "judge-arena/judge-arena-dataset": "004c4a992956eeefffd36b63ade470f32fd0a582",
     # MT-Bench questions (LMSYS Space).
     "lmsys/mt-bench": "a4b674ca573c24143824ac7f60d9173e7081e37d",
     # Multilingual fluency contexts (generated by scripts/fluency/generate_fluency.py).

judgearena/datasets/__init__.py

@@ -9,12 +9,22 @@
     split_m_arena_hard_dataset,
 )
 from judgearena.log import get_logger
+from judgearena.tasks.registry import get_packaged_task
 
 logger = get_logger(__name__)
 
 
 def load_instructions(dataset: str, n_instructions: int | None = None) -> pd.DataFrame:
-    if dataset == "mt-bench":
+    resolved_task = get_packaged_task(dataset)
+    if resolved_task is not None:
+        from judgearena import utils as judgearena_utils
+        from judgearena.datasets.judgearena_tables import load_task_instructions
+
+        df_instructions = load_task_instructions(
+            resolved_task, judgearena_utils.data_root / "tables"
+        )
+
+    elif dataset == "mt-bench":
         from judgearena.datasets.mt_bench import load_mt_bench
 
         df_instructions = load_mt_bench()
@@ -46,7 +56,6 @@ def load_instructions(dataset: str, n_instructions: int | None = None) -> pd.Dat
 
     else:
         assert dataset in [
-            "alpaca-eval",
             "arena-hard-v0.1",
             "arena-hard-v2.0",
         ]

judgearena/datasets/judgearena_tables.py

@@ -0,0 +1,70 @@
+"""Dataset adapter for JudgeArena's packaged instruction/output tables."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pandas as pd
+from huggingface_hub import snapshot_download
+
+from judgearena.tasks.schema import HuggingFaceDatasetSource, ResolvedTaskSpec
+
+
+def download_task_sources(task: ResolvedTaskSpec, local_dir: Path) -> None:
+    """Download every Hugging Face source declared by a table-backed task."""
+    if task.spec.dataset.adapter != "judgearena_tables":
+        raise ValueError(
+            f"Task {task.task!r} uses dataset adapter "
+            f"{task.spec.dataset.adapter!r}, not 'judgearena_tables'."
+        )
+    local_dir.mkdir(exist_ok=True, parents=True)
+    for name, source in task.spec.dataset.sources.items():
+        if not isinstance(source, HuggingFaceDatasetSource):
+            raise ValueError(
+                f"Dataset source {name!r} for task {task.task!r} is not supported "
+                "by the 'judgearena_tables' adapter."
+            )
+        snapshot_download(
+            repo_id=source.repo_id,
+            repo_type="dataset",
+            revision=source.revision,
+            allow_patterns=list(source.allow_patterns) or None,
+            local_dir=local_dir,
+            force_download=False,
+        )
+
+
+def load_task_instructions(
+    task: ResolvedTaskSpec, local_tables_path: Path
+) -> pd.DataFrame:
+    """Load a task's table and map its declared fields to runner names."""
+    download_task_sources(task, local_tables_path)
+    path = local_tables_path / "instructions" / f"{task.task}.csv"
+    if not path.exists():
+        raise FileNotFoundError(f"Instruction table not found at {path}")
+    df = pd.read_csv(path)
+
+    fields = task.spec.dataset.fields
+    field_mapping = {
+        fields.id: "instruction_index",
+        fields.instruction: "instruction",
+    }
+    if fields.category is not None:
+        field_mapping[fields.category] = "category"
+    missing = sorted(set(field_mapping) - set(df.columns))
+    if missing:
+        raise ValueError(
+            f"Task {task.task!r} is missing declared dataset fields: {missing}."
+        )
+    return df.rename(columns=field_mapping)
+
+
+def load_task_model_outputs(
+    task: ResolvedTaskSpec, local_tables_path: Path
+) -> pd.DataFrame | None:
+    """Load optional pre-generated model outputs for a table-backed task."""
+    download_task_sources(task, local_tables_path)
+    path = local_tables_path / "model_outputs" / f"{task.task}.csv.zip"
+    if not path.exists():
+        return None
+    return pd.read_csv(path)