diff --git a/judgearena/benchmarks/mt_bench/runner.py b/judgearena/benchmarks/mt_bench/runner.py index 5e05e97..69ff5f0 100644 --- a/judgearena/benchmarks/mt_bench/runner.py +++ b/judgearena/benchmarks/mt_bench/runner.py @@ -19,6 +19,7 @@ ) from judgearena.benchmarks.mt_bench.preset_judging import judge_mt_bench_with_preset from judgearena.benchmarks.pairwise.baselines import native_pairwise_baseline +from judgearena.benchmarks.pairwise.scoring import PAIRWISE_SCORERS from judgearena.datasets import load_instructions from judgearena.datasets.mt_bench import ( load_mt_bench_model_answers, @@ -27,11 +28,9 @@ from judgearena.log import get_logger from judgearena.models import is_thinking_model, make_model from judgearena.prompts.registry import ResolvedJudgePrompt, resolve_run_judge_prompt -from judgearena.tasks.registry import get_packaged_task from judgearena.tasks.schema import MTBenchProtocol from judgearena.utils import ( cache_function_dataframe, - compute_pref_summary, generation_cache_token, ) from judgearena.utils.eval import BattleReport, _compute_grouped_stats @@ -40,13 +39,7 @@ if TYPE_CHECKING: from judgearena.config import RunConfig - - -def _task_protocol(task_id: str) -> MTBenchProtocol: - task = get_packaged_task(task_id) - if task is None or not isinstance(task.spec.protocol, MTBenchProtocol): - raise ValueError(f"Task {task_id!r} does not define an MT-Bench protocol.") - return task.spec.protocol + from judgearena.tasks.schema import ResolvedTaskSpec def _align_mt_bench_completions( @@ -90,10 +83,10 @@ def _build_mt_bench_generation_kwargs( def _generate_mt_bench_completions( cfg: RunConfig, + protocol: MTBenchProtocol, questions_df: pd.DataFrame, ) -> tuple[pd.DataFrame, pd.DataFrame]: cache_prefix = cfg.task - protocol = _task_protocol(cfg.task) def _run_generation( model_name: str, *, generation_kwargs: dict[str, object] @@ -198,6 +191,7 @@ def _save_mt_bench_results( def _finalize_mt_bench_run( *, cfg: RunConfig, + protocol: MTBenchProtocol, res_folder: Path, result_name: str, prefs: pd.Series, @@ -210,7 +204,8 @@ def _finalize_mt_bench_run( started_at_utc: datetime, extra_result_fields: dict[str, object] | None = None, ) -> pd.Series: - stats = compute_pref_summary(prefs) + scorer = PAIRWISE_SCORERS[protocol.scoring.adapter] + stats = scorer.summarize(prefs) report = BattleReport( task=cfg.task, model_a=cfg.model.name, @@ -253,6 +248,7 @@ def _finalize_mt_bench_run( def _run_mt_bench_fastchat( *, cfg: RunConfig, + protocol: MTBenchProtocol, res_folder: Path, result_name: str, questions_df: pd.DataFrame, @@ -260,10 +256,8 @@ def _run_mt_bench_fastchat( completions_b: pd.DataFrame, judge_chat_model, resolved_prompt: ResolvedJudgePrompt, - fastchat_prompt_preset: str, started_at_utc: datetime, ) -> pd.Series: - protocol = _task_protocol(cfg.task) prefs, annotations, combined_metadata, num_inconsistent = ( judge_mt_bench_pairwise_fastchat( judge_chat_model=judge_chat_model, @@ -278,12 +272,13 @@ def _run_mt_bench_fastchat( truncate_input_chars=cfg.generation.truncate_judge_input_chars, use_tqdm=cfg.run.use_tqdm, reference_categories=protocol.judge.reference_categories, - prompt_preset=fastchat_prompt_preset, + prompt_preset=protocol.judge.fastchat_prompt_preset, strip_thinking_before_judging=cfg.judge.strip_thinking_before_judging, ) ) return _finalize_mt_bench_run( cfg=cfg, + protocol=protocol, res_folder=res_folder, result_name=result_name, prefs=prefs, @@ -301,6 +296,7 @@ def _run_mt_bench_fastchat( def _run_mt_bench_preset( *, cfg: RunConfig, + protocol: MTBenchProtocol, res_folder: Path, result_name: str, questions_df: pd.DataFrame, @@ -310,7 +306,6 @@ def _run_mt_bench_preset( resolved_prompt: ResolvedJudgePrompt, started_at_utc: datetime, ) -> pd.Series: - protocol = _task_protocol(cfg.task) prefs, annotations, combined_metadata = judge_mt_bench_with_preset( judge_chat_model=judge_chat_model, judge_model=cfg.judge.model, @@ -332,6 +327,7 @@ def _run_mt_bench_preset( ) return _finalize_mt_bench_run( cfg=cfg, + protocol=protocol, res_folder=res_folder, result_name=result_name, prefs=prefs, @@ -345,10 +341,12 @@ def _run_mt_bench_preset( ) -def run_mt_bench_benchmark(cfg: RunConfig): +def run_mt_bench_benchmark(cfg: RunConfig, task: ResolvedTaskSpec | None = None): """Run the registered MT-Bench generation, judging, and reporting lifecycle.""" run_started_at = datetime.now(UTC) - protocol = _task_protocol(cfg.task) + protocol = task.spec.protocol if task is not None else None + if not isinstance(protocol, MTBenchProtocol): + raise ValueError(f"Task {cfg.task!r} does not define an MT-Bench protocol.") if cfg.model.baseline is None: baseline = native_pairwise_baseline(cfg.task) cfg.model.baseline = baseline if isinstance(baseline, str) else None @@ -357,7 +355,6 @@ def run_mt_bench_benchmark(cfg: RunConfig): f"--model_B is required for dataset '{cfg.task}'; " "no dataset-native baseline registered." ) - result_name = ( f"{cfg.task}-{cfg.model.name}-{cfg.model.baseline}-{cfg.judge.model}-" f"{cfg.judge.swap_mode}" @@ -378,6 +375,7 @@ def run_mt_bench_benchmark(cfg: RunConfig): ) completions_a, completions_b = _generate_mt_bench_completions( cfg=cfg, + protocol=protocol, questions_df=questions_df, ) resolved_prompt = resolve_run_judge_prompt(cfg.task, cfg.judge, multi_turn=True) @@ -398,6 +396,7 @@ def run_mt_bench_benchmark(cfg: RunConfig): if resolved_prompt.delegated: return _run_mt_bench_fastchat( cfg=cfg, + protocol=protocol, res_folder=res_folder, result_name=result_name, questions_df=questions_df, @@ -405,11 +404,11 @@ def run_mt_bench_benchmark(cfg: RunConfig): completions_b=completions_b, judge_chat_model=judge_chat_model, resolved_prompt=resolved_prompt, - fastchat_prompt_preset=protocol.judge.fastchat_prompt_preset, started_at_utc=run_started_at, ) return _run_mt_bench_preset( cfg=cfg, + protocol=protocol, res_folder=res_folder, result_name=result_name, questions_df=questions_df, diff --git a/judgearena/benchmarks/pairwise/baselines.py b/judgearena/benchmarks/pairwise/baselines.py index 8ea3bbd..899d1b6 100644 --- a/judgearena/benchmarks/pairwise/baselines.py +++ b/judgearena/benchmarks/pairwise/baselines.py @@ -3,9 +3,103 @@ from __future__ import annotations from collections.abc import Mapping +from dataclasses import dataclass + +import pandas as pd from judgearena.tasks.registry import get_packaged_task -from judgearena.tasks.schema import CategoryDefaultsBaseline, TaskDefaultBaseline +from judgearena.tasks.schema import ( + CategoryDefaultsBaseline, + ResolvedTaskSpec, + TaskDefaultBaseline, +) + + +@dataclass(frozen=True) +class BaselinePlan: + """Row-aligned baseline assignment for model B.""" + + baseline_by_index: pd.Series + + @classmethod + def flat(cls, model: str, *, index: pd.Index) -> BaselinePlan: + return cls( + baseline_by_index=pd.Series(model, index=index, name="model_B", dtype=str) + ) + + @classmethod + def per_row(cls, series: pd.Series) -> BaselinePlan: + return cls(baseline_by_index=series.astype(str).rename("model_B")) + + @property + def unique_models(self) -> list[str]: + return sorted(self.baseline_by_index.dropna().unique().tolist()) + + @property + def is_single_model(self) -> bool: + return len(self.unique_models) == 1 + + @property + def single_model(self) -> str: + if not self.is_single_model: + raise ValueError( + "BaselinePlan is per-row; use baseline_by_index for row-level lookups." + ) + return self.unique_models[0] + + @property + def display_name(self) -> str: + return ( + self.single_model if self.is_single_model else "+".join(self.unique_models) + ) + + def aligned_to(self, index: pd.Index) -> pd.Series: + return self.baseline_by_index.loc[index] + + +def resolve_baseline_plan( + *, + task_id: str, + task: ResolvedTaskSpec | None, + runtime_baseline: str | None, + instructions: pd.DataFrame, +) -> BaselinePlan: + """Resolve a runtime override or the baseline declared by a task.""" + if runtime_baseline is not None: + return BaselinePlan.flat(runtime_baseline, index=instructions.index) + + if task is None: + raise ValueError( + f"model.baseline is required for task {task_id!r}; no task baseline " + "is registered." + ) + + baseline = task.spec.protocol.baseline + if isinstance(baseline, TaskDefaultBaseline): + return BaselinePlan.flat(baseline.reference_id, index=instructions.index) + + if isinstance(baseline, CategoryDefaultsBaseline): + if "category" not in instructions.columns: + raise ValueError( + f"{task_id} requires a 'category' column for per-category " + "baseline routing; re-run dataset download to regenerate the " + "instructions table." + ) + per_row = instructions["category"].map(baseline.references) + if per_row.isna().any(): + unknown = sorted( + instructions.loc[per_row.isna(), "category"].unique().tolist() + ) + raise ValueError( + f"Unknown baseline categories for {task_id}: {unknown}. " + f"Known: {sorted(baseline.references)}" + ) + return BaselinePlan.per_row(per_row) + + raise ValueError( + f"model.baseline is required for task {task_id!r}; its " + f"{baseline.strategy!r} baseline policy does not provide a model." + ) def native_pairwise_baseline(task: str) -> str | Mapping[str, str] | None: diff --git a/judgearena/benchmarks/pairwise/runner.py b/judgearena/benchmarks/pairwise/runner.py index 09937f4..9f7d428 100644 --- a/judgearena/benchmarks/pairwise/runner.py +++ b/judgearena/benchmarks/pairwise/runner.py @@ -3,8 +3,6 @@ and then evaluates them using a judge model. """ -from collections.abc import Mapping -from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path from typing import TYPE_CHECKING @@ -13,17 +11,22 @@ from judgearena.artifacts import prepare_run_directory, write_run_metadata_safely from judgearena.benchmarks.execution import build_generation_kwargs, build_judge -from judgearena.benchmarks.pairwise.baselines import native_pairwise_baseline +from judgearena.benchmarks.pairwise.baselines import resolve_baseline_plan +from judgearena.benchmarks.pairwise.scoring import ( + DEFAULT_PAIRWISE_SCORER, + PAIRWISE_SCORERS, +) from judgearena.datasets import load_instructions from judgearena.datasets.fluency import is_fluency_task as task_is_fluency from judgearena.datasets.fluency import load_fluency_contexts +from judgearena.datasets.pairwise import PairwiseTaskData, load_pairwise_task_data from judgearena.evaluate import judge_and_parse_prefs, resolve_run_judge_prompt from judgearena.generate import generate_base, generate_instructions from judgearena.log import get_logger from judgearena.tasks.registry import get_packaged_task +from judgearena.tasks.schema import ResolvedTaskSpec from judgearena.utils import ( cache_function_dataframe, - compute_pref_summary, data_root, download_hf, generation_cache_token, @@ -37,34 +40,19 @@ logger = get_logger(__name__) -def try_load_dataset_completions( +def _try_load_legacy_dataset_completions( dataset: str, model: str, n_instructions: int | None ) -> pd.DataFrame | None: - """Try loading pre-existing completions from the dataset. - - Some datasets (e.g. alpaca-eval) ship with completions for well-known - models such as ``gpt4_1106_preview``. When ``model`` matches a column in - ``model_outputs/{dataset}.csv.zip``, those completions are returned - directly so that no model instantiation / generation is needed. + """Try loading pre-existing completions for an unregistered legacy task. - Returns a DataFrame with columns ``completion`` and ``instruction_index``, - or ``None`` when no pre-existing completions are found. + Registered tasks load outputs through ``PairwiseTaskData`` instead. """ local_path_tables = data_root / "tables" - resolved_task = get_packaged_task(dataset) - if resolved_task is not None: - from judgearena.datasets.registry import resolve_dataset_adapter - - adapter = resolve_dataset_adapter(resolved_task.spec.dataset.adapter) - df_outputs = adapter.load_model_outputs(resolved_task, local_path_tables) - if df_outputs is None: - return None - else: - download_hf(name=dataset, local_path=local_path_tables) - output_path = local_path_tables / "model_outputs" / f"{dataset}.csv.zip" - if not output_path.exists(): - return None - df_outputs = read_df(output_path) + download_hf(name=dataset, local_path=local_path_tables) + output_path = local_path_tables / "model_outputs" / f"{dataset}.csv.zip" + if not output_path.exists(): + return None + df_outputs = read_df(output_path) df_outputs.loc[:, "output"] = df_outputs.loc[:, "output"].fillna("") df_outputs = df_outputs.pivot_table( index="instruction_index", columns="model", values="output", aggfunc="last" @@ -85,82 +73,7 @@ def try_load_dataset_completions( ) -@dataclass(frozen=True) -class BaselinePlan: - """Row-aligned baseline assignment for `--model_B`.""" - - baseline_by_index: pd.Series - - @classmethod - def flat(cls, model: str, *, index: pd.Index) -> "BaselinePlan": - return cls( - baseline_by_index=pd.Series(model, index=index, name="model_B", dtype=str) - ) - - @classmethod - def per_row(cls, series: pd.Series) -> "BaselinePlan": - return cls(baseline_by_index=series.astype(str).rename("model_B")) - - @property - def unique_models(self) -> list[str]: - return sorted(self.baseline_by_index.dropna().unique().tolist()) - - @property - def is_flat(self) -> bool: - return len(self.unique_models) == 1 - - @property - def single_model(self) -> str: - if not self.is_flat: - raise ValueError( - "BaselinePlan is per-row; use baseline_by_index for row-level lookups." - ) - return self.unique_models[0] - - @property - def display_name(self) -> str: - return self.single_model if self.is_flat else "+".join(self.unique_models) - - def aligned_to(self, index: pd.Index) -> pd.Series: - return self.baseline_by_index.loc[index] - - -def _resolve_baseline_plan( - *, task: str, model_b: str | None, instructions_df: pd.DataFrame -) -> BaselinePlan: - """Resolve explicit or dataset-native baseline assignment.""" - if model_b is not None: - return BaselinePlan.flat(model_b, index=instructions_df.index) - - native = native_pairwise_baseline(task) - if native is None: - raise ValueError( - f"model.baseline is required for task '{task}'; no dataset-native " - "baseline is registered." - ) - if isinstance(native, str): - return BaselinePlan.flat(native, index=instructions_df.index) - if isinstance(native, Mapping): - if "category" not in instructions_df.columns: - raise ValueError( - f"{task} requires a 'category' column for per-category " - "baseline routing; re-run dataset download to regenerate the " - "instructions table." - ) - per_row = instructions_df["category"].map(native) - if per_row.isna().any(): - unknown = sorted( - instructions_df.loc[per_row.isna(), "category"].unique().tolist() - ) - raise ValueError( - f"Unknown Arena-Hard categories for {task}: {unknown}. " - f"Known: {sorted(native.keys())}" - ) - return BaselinePlan.per_row(per_row) - raise ValueError(f"Unsupported baseline shape for dataset '{task}'.") - - -def run_pairwise(cfg: "RunConfig"): +def run_pairwise(cfg: "RunConfig", resolved_task: ResolvedTaskSpec | None = None): """ 1) take as input: * task (dataset), make sure instruct-completion works @@ -181,7 +94,16 @@ def run_pairwise(cfg: "RunConfig"): # Currrently, we run context evaluation is_fluency_task = task_is_fluency(cfg.task) - if is_fluency_task: + resolved_task = resolved_task or get_packaged_task(cfg.task) + task_data: PairwiseTaskData | None = None + if resolved_task is not None: + task_data = load_pairwise_task_data( + resolved_task, + n_instructions=cfg.generation.n_instructions, + ) + instructions_df = task_data.instructions + instructions = instructions_df.loc[:, "instruction"] + elif is_fluency_task: # if cfg.task = "fluency-french", we map to the "French" config of # https://huggingface.co/datasets/geoalgo/multilingual-fluency instructions = load_fluency_contexts(data_root, cfg.task) @@ -202,8 +124,11 @@ def run_pairwise(cfg: "RunConfig"): instructions_df = instructions_df.head(n_instructions) instructions = instructions.head(n_instructions) - baseline_plan = _resolve_baseline_plan( - task=cfg.task, model_b=cfg.model.baseline, instructions_df=instructions_df + baseline_plan = resolve_baseline_plan( + task_id=cfg.task, + task=resolved_task, + runtime_baseline=cfg.model.baseline, + instructions=instructions_df, ) name = f"{cfg.task}-{cfg.model.name}-{baseline_plan.display_name}-{cfg.judge.model}" @@ -246,10 +171,20 @@ def _run_generation( def _align_completion_series(df: pd.DataFrame) -> pd.Series: return df.set_index("instruction_index").loc[instructions.index, "completion"] + def _preloaded_completions(model_spec: str) -> pd.Series | None: + """Aligned completions shipped with the dataset, if any.""" + if task_data is not None: + preloaded = task_data.model_completion(model_spec) + return None if preloaded is None else preloaded.loc[instructions.index] + legacy = _try_load_legacy_dataset_completions( + cfg.task, model_spec, n_instructions + ) + return None if legacy is None else _align_completion_series(legacy) + def _load_or_generate_completions(model_spec: str, *, role: str) -> pd.Series: - preloaded = try_load_dataset_completions(cfg.task, model_spec, n_instructions) + preloaded = _preloaded_completions(model_spec) if preloaded is not None: - return _align_completion_series(preloaded) + return preloaded # Fold the resolved generation kwargs into the cache key so that changing # any sampling param (temperature, seed, top_p/k, max_tokens, ...) busts # the cached completions instead of silently reusing a stale run. @@ -268,7 +203,7 @@ def _load_or_generate_completions(model_spec: str, *, role: str) -> pd.Series: completions_A = _load_or_generate_completions(cfg.model.name, role="A") baseline_per_index = baseline_plan.aligned_to(instructions.index) - if baseline_plan.is_flat: + if baseline_plan.is_single_model: completions_B = _load_or_generate_completions( baseline_plan.single_model, role="B" ) @@ -334,8 +269,12 @@ def _load_or_generate_completions(model_spec: str, *, role: str) -> pd.Series: df.to_csv(res_folder / f"{name}-annotations.csv", index=False) - # compute and report statistics - summary = compute_pref_summary(prefs) + scorer = PAIRWISE_SCORERS[ + resolved_task.spec.protocol.scoring.adapter + if resolved_task is not None + else DEFAULT_PAIRWISE_SCORER + ] + summary = scorer.summarize(prefs) report = BattleReport( task=cfg.task, @@ -347,7 +286,9 @@ def _load_or_generate_completions(model_spec: str, *, role: str) -> pd.Series: result_folder=str(res_folder), preferences=prefs.tolist(), metadata={ - "baseline_assignment": "per-row" if not baseline_plan.is_flat else "flat", + "baseline_assignment": "per-row" + if not baseline_plan.is_single_model + else "flat", "baseline_models": baseline_plan.unique_models, **resolved_prompt.metadata(), "strip_thinking_before_judging": cfg.judge.strip_thinking_before_judging, diff --git a/judgearena/benchmarks/pairwise/scoring.py b/judgearena/benchmarks/pairwise/scoring.py new file mode 100644 index 0000000..7ae21ec --- /dev/null +++ b/judgearena/benchmarks/pairwise/scoring.py @@ -0,0 +1,31 @@ +"""Runtime scoring adapters for pairwise preference tasks.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass + +import pandas as pd + +from judgearena.utils.eval import PrefSummary, compute_pref_summary + + +@dataclass(frozen=True) +class PairwiseScorer: + """Scoring implementation and the semantics of its primary metric.""" + + primary_metric: str + higher_is_better: bool + summarize: Callable[[pd.Series], PrefSummary] + + +PAIRWISE_SCORERS = { + "pairwise_win_rate": PairwiseScorer( + primary_metric="winrate", + higher_is_better=True, + summarize=compute_pref_summary, + ) +} + +# Used by legacy tasks that have no YAML definition to declare a scorer. +DEFAULT_PAIRWISE_SCORER = "pairwise_win_rate" diff --git a/judgearena/benchmarks/registry.py b/judgearena/benchmarks/registry.py index 7003910..83dc514 100644 --- a/judgearena/benchmarks/registry.py +++ b/judgearena/benchmarks/registry.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Protocol from judgearena.tasks.registry import get_packaged_task +from judgearena.tasks.schema import ResolvedTaskSpec if TYPE_CHECKING: from judgearena.config import RunConfig @@ -14,7 +15,7 @@ class BenchmarkRunner(Protocol): """Callable implemented by a benchmark-specific evaluation module.""" - def __call__(self, cfg: RunConfig, /) -> object: ... + def __call__(self, cfg: RunConfig, task: ResolvedTaskSpec | None, /) -> object: ... @dataclass(frozen=True) @@ -33,6 +34,14 @@ def supports(self, task: str) -> bool: return self.tasks is None or task in self.tasks +@dataclass(frozen=True) +class ResolvedBenchmark: + """Runner selection and the task definition resolved during dispatch.""" + + adapter: BenchmarkAdapter + task: ResolvedTaskSpec | None + + def benchmark_adapters() -> tuple[BenchmarkAdapter, ...]: """Return registered benchmark implementations, specific first.""" from judgearena.benchmarks.mt_bench.runner import run_mt_bench_benchmark @@ -44,18 +53,23 @@ def benchmark_adapters() -> tuple[BenchmarkAdapter, ...]: ) -def resolve_benchmark_adapter(task: str) -> BenchmarkAdapter: - """Resolve a YAML-selected runner, then fall back for unmigrated tasks.""" +def resolve_benchmark(task: str) -> ResolvedBenchmark: + """Resolve a runner and task definition with one registry lookup.""" adapters = benchmark_adapters() resolved = get_packaged_task(task) if resolved is not None: runner_id = resolved.spec.protocol.runner for adapter in adapters: if adapter.name == runner_id: - return adapter + return ResolvedBenchmark(adapter=adapter, task=resolved) raise ValueError(f"Task {task!r} selects unavailable runner {runner_id!r}.") for adapter in adapters: if adapter.supports(task): - return adapter + return ResolvedBenchmark(adapter=adapter, task=None) raise ValueError(f"No generate-and-evaluate adapter supports task {task!r}.") + + +def resolve_benchmark_adapter(task: str) -> BenchmarkAdapter: + """Return only the selected adapter for compatibility and inspection.""" + return resolve_benchmark(task).adapter diff --git a/judgearena/benchmarks/runner.py b/judgearena/benchmarks/runner.py index 103bd85..dddd9f6 100644 --- a/judgearena/benchmarks/runner.py +++ b/judgearena/benchmarks/runner.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING -from judgearena.benchmarks.registry import resolve_benchmark_adapter +from judgearena.benchmarks.registry import resolve_benchmark from judgearena.log import get_logger if TYPE_CHECKING: @@ -15,6 +15,6 @@ def run_benchmark(cfg: RunConfig) -> object: """Run a task through its registered benchmark adapter.""" - adapter = resolve_benchmark_adapter(cfg.task) - logger.info("Using %s benchmark adapter for %s.", adapter.name, cfg.task) - return adapter.runner(cfg) + resolved = resolve_benchmark(cfg.task) + logger.info("Using %s benchmark adapter for %s.", resolved.adapter.name, cfg.task) + return resolved.adapter.runner(cfg, resolved.task) diff --git a/judgearena/datasets/pairwise.py b/judgearena/datasets/pairwise.py new file mode 100644 index 0000000..f9c4e4c --- /dev/null +++ b/judgearena/datasets/pairwise.py @@ -0,0 +1,97 @@ +"""Canonical data contract for registered single-turn pairwise tasks.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +import pandas as pd + +from judgearena.datasets.registry import resolve_dataset_adapter +from judgearena.log import get_logger +from judgearena.tasks.schema import PairwiseProtocol, ResolvedTaskSpec +from judgearena.utils import data_root + +logger = get_logger(__name__) + + +@dataclass(frozen=True) +class PairwiseTaskData: + """Normalized instructions and optional pre-generated model outputs. + + ``model_outputs`` is ``None`` when the dataset ships no pre-generated + completions; the runner generates whatever it cannot find here. + """ + + instructions: pd.DataFrame + model_outputs: pd.DataFrame | None = None + + def __post_init__(self) -> None: + if self.model_outputs is None: + return + required = {"instruction_index", "model", "output"} + missing = sorted(required - set(self.model_outputs.columns)) + if missing: + raise ValueError( + f"Pairwise model outputs are missing canonical columns: {missing}." + ) + + def model_completion(self, model: str) -> pd.Series | None: + """Return model completions aligned to the instruction index.""" + if self.model_outputs is None: + return None + + outputs = self.model_outputs.loc[ + self.model_outputs["model"] == model, + ["instruction_index", "output"], + ].copy() + if outputs.empty: + return None + + outputs["output"] = outputs["output"].fillna("") + completions = ( + outputs.drop_duplicates("instruction_index", keep="last") + .set_index("instruction_index")["output"] + .sort_index() + ) + logger.info("Found pre-existing completions for model %r.", model) + return completions.loc[self.instructions.index].rename("completion") + + +def load_pairwise_task_data( + task: ResolvedTaskSpec, + *, + n_instructions: int | None = None, + local_tables_path: Path | None = None, +) -> PairwiseTaskData: + """Load one registered task through its declared dataset adapter.""" + if not isinstance(task.spec.protocol, PairwiseProtocol): + raise ValueError(f"Task {task.task!r} does not use the pairwise protocol.") + + tables_path = local_tables_path or data_root / "tables" + adapter = resolve_dataset_adapter(task.spec.dataset.adapter) + instructions = adapter.load_instructions(task, tables_path) + if "instruction_index" in instructions.columns: + instructions = instructions.set_index("instruction_index") + if instructions.index.name != "instruction_index": + raise ValueError( + f"Dataset adapter {adapter.name!r} must provide 'instruction_index'." + ) + if "instruction" not in instructions.columns: + raise ValueError( + f"Dataset adapter {adapter.name!r} must provide 'instruction'." + ) + if instructions.index.has_duplicates: + raise ValueError( + f"Dataset adapter {adapter.name!r} returned duplicate instruction IDs." + ) + + instructions = instructions.sort_index() + if n_instructions is not None: + instructions = instructions.head(n_instructions) + logger.info("Loaded %d instructions for %s.", len(instructions), task.task) + + return PairwiseTaskData( + instructions=instructions, + model_outputs=adapter.load_model_outputs(task, tables_path), + ) diff --git a/judgearena/tasks/README.md b/judgearena/tasks/README.md new file mode 100644 index 0000000..9ef9191 --- /dev/null +++ b/judgearena/tasks/README.md @@ -0,0 +1,113 @@ +# Task definitions + +Task YAML describes **what benchmark is being run**. A run config describes +**how one experiment runs it**. + +- Keep stable benchmark behavior in the task: pinned data, field mappings, + baseline policy, judge protocol, and scoring definition. +- Keep experiment choices in `RunConfig`: candidate and judge models, sampling + parameters, instruction limits, caching, and supported overrides. +- Prefer a new task version when changing data or evaluation behavior. Changing + only the model or runtime settings does not create a new task. + +## Layout + +```text +tasks/ +├── definitions/ # Packaged task YAML, grouped by benchmark family +├── schema/ # Typed YAML contracts, separated by responsibility +├── loader.py # Inheritance, strict YAML loading, and stable hashes +├── registry.py # Discovery, task lookup, and component validation +└── cli.py # `judgearena tasks ...` commands +``` + +Executable behavior lives outside the definitions: + +- `judgearena/datasets/` downloads and normalizes upstream data. +- `judgearena/benchmarks/` runs the selected evaluation workflow. +- `judgearena/prompts/` owns reusable judge prompt presets. + +## Add a task using existing components + +Create a public YAML file under `definitions//`. Files beginning with +`_` are private bases and are not runnable tasks. + +```yaml +schema_version: 1 +task: example-pairwise +task_version: 1 +description: Pairwise evaluation on the Example instruction set. +tags: [pairwise, instruction-following] + +dataset: + adapter: judgearena_tables + sources: + examples: + type: huggingface_dataset + repo_id: organization/example + revision: "0123456789abcdef0123456789abcdef01234567" + fields: + id: instruction_index + instruction: instruction + +protocol: + runner: pairwise + generation: + mode: single_turn_chat + baseline: + strategy: task_default + reference_id: example-baseline + judge: + default_prompt: default + scoring: + adapter: pairwise_win_rate +``` + +Reuse a private `_base.yaml` with `extends: _base.yaml` when several versions +share meaningful settings. Avoid inheritance merely to save a few lines. + +Validate the result before running it: + +```bash +judgearena tasks validate example-pairwise +judgearena tasks show example-pairwise --resolved +``` + +The resolved view includes inherited values and hashes of every YAML resource. + +## Add new behavior + +Most new tasks should reuse the existing `pairwise` protocol and need no new +runner code. + +Keep task YAML boring: declarative facts belong in YAML, while downloading, +format conversion, and scoring algorithms belong in Python. + +The judge prompt preset owns the expected output format and its parser. The +scoring adapter owns the calculation, primary metric, and metric direction; +task YAML only selects those components by ID. + +If an upstream dataset has a new format, implement a dataset adapter under +`judgearena/datasets/` and register its ID in both the dataset registry and the +task adapter catalog. The adapter must return JudgeArena's canonical columns; +the common runner should never check a task name. + +If a benchmark needs a genuinely different evaluation algorithm: + +1. Add its typed protocol under `schema/`. +2. Add that protocol to `ProtocolSpec` in `schema/task.py`. +3. Implement and register its benchmark runner. +4. Keep algorithm-specific fields in that protocol, as MT-Bench does in + `MTBenchProtocol` and `MTBenchJudgeSpec`. + +Adding a schema field makes it valid in task YAML; it does not automatically +make it a run-config or CLI option. Runtime overrides must be explicitly added +to `RunConfig` and resolved with this precedence: + +```text +CLI > run config YAML > task YAML > code default +``` + +Dataset revisions and evaluation definitions should normally remain +task-owned. Expose overrides only for settings users are expected to change +between experiments. diff --git a/judgearena/tasks/definitions/alpaca_eval/alpaca-eval.yaml b/judgearena/tasks/definitions/alpaca_eval/alpaca-eval.yaml index fdd972e..f624a09 100644 --- a/judgearena/tasks/definitions/alpaca_eval/alpaca-eval.yaml +++ b/judgearena/tasks/definitions/alpaca_eval/alpaca-eval.yaml @@ -26,13 +26,10 @@ protocol: allow_runtime_override: true judge: default_prompt: default - parser: pairwise_preference default_swap_mode: fixed allowed_swap_modes: [fixed, both] scoring: adapter: pairwise_win_rate - primary_metric: winrate - higher_is_better: true metadata: reference_implementation: https://github.com/tatsu-lab/alpaca_eval diff --git a/judgearena/tasks/definitions/arena_hard/_base.yaml b/judgearena/tasks/definitions/arena_hard/_base.yaml index e4b2810..fd865b9 100644 --- a/judgearena/tasks/definitions/arena_hard/_base.yaml +++ b/judgearena/tasks/definitions/arena_hard/_base.yaml @@ -19,13 +19,10 @@ protocol: mode: single_turn_chat judge: default_prompt: default - parser: pairwise_preference default_swap_mode: fixed allowed_swap_modes: [fixed, both] scoring: adapter: pairwise_win_rate - primary_metric: winrate - higher_is_better: true metadata: reference_implementation: https://github.com/lmarena/arena-hard-auto diff --git a/judgearena/tasks/definitions/m_arena_hard/_base.yaml b/judgearena/tasks/definitions/m_arena_hard/_base.yaml index 12eb149..df2f8e7 100644 --- a/judgearena/tasks/definitions/m_arena_hard/_base.yaml +++ b/judgearena/tasks/definitions/m_arena_hard/_base.yaml @@ -19,13 +19,10 @@ protocol: mode: single_turn_chat judge: default_prompt: default - parser: pairwise_preference default_swap_mode: fixed allowed_swap_modes: [fixed, both] scoring: adapter: pairwise_win_rate - primary_metric: winrate - higher_is_better: true variants: selector: language diff --git a/judgearena/tasks/definitions/mt_bench/mt-bench.yaml b/judgearena/tasks/definitions/mt_bench/mt-bench.yaml index 099ee12..77cc430 100644 --- a/judgearena/tasks/definitions/mt_bench/mt-bench.yaml +++ b/judgearena/tasks/definitions/mt_bench/mt-bench.yaml @@ -44,7 +44,6 @@ protocol: allow_runtime_override: true judge: default_prompt: fastchat-pairwise - parser: fastchat_pairwise_verdict default_swap_mode: fixed allowed_swap_modes: [fixed, both] turns_mode: both @@ -53,8 +52,6 @@ protocol: reference_categories: [math, reasoning, coding, arena-hard-200] scoring: adapter: pairwise_win_rate - primary_metric: winrate - higher_is_better: true metadata: reference_implementation: https://github.com/lm-sys/FastChat/tree/master/fastchat/llm_judge diff --git a/judgearena/tasks/registry.py b/judgearena/tasks/registry.py index 9cd158f..66e2c4f 100644 --- a/judgearena/tasks/registry.py +++ b/judgearena/tasks/registry.py @@ -16,6 +16,7 @@ import yaml from pydantic import ValidationError +from judgearena.benchmarks.pairwise.scoring import PAIRWISE_SCORERS from judgearena.log import get_logger from judgearena.prompts.registry import JUDGE_PROMPT_PRESETS from judgearena.tasks.schema import ( @@ -249,10 +250,7 @@ class AdapterCatalog: {"arena_hard", "judgearena_tables", "m_arena_hard", "mt_bench"} ) prompts: frozenset[str] = frozenset(JUDGE_PROMPT_PRESETS) - parsers: frozenset[str] = frozenset( - {"fastchat_pairwise_verdict", "pairwise_preference"} - ) - scorers: frozenset[str] = frozenset({"pairwise_win_rate"}) + scorers: frozenset[str] = frozenset(PAIRWISE_SCORERS) def load_tasks( @@ -301,7 +299,6 @@ def _validate_adapter_ids(resolved: ResolvedTaskSpec, adapters: AdapterCatalog) "runner": (spec.protocol.runner, adapters.runners), "dataset adapter": (spec.dataset.adapter, adapters.datasets), "prompt": (spec.protocol.judge.default_prompt, adapters.prompts), - "parser": (spec.protocol.judge.parser, adapters.parsers), "scorer": (spec.protocol.scoring.adapter, adapters.scorers), } for kind, (adapter_id, available) in references.items(): diff --git a/judgearena/tasks/schema.py b/judgearena/tasks/schema.py deleted file mode 100644 index 637361a..0000000 --- a/judgearena/tasks/schema.py +++ /dev/null @@ -1,312 +0,0 @@ -"""Typed contract for task YAML; this module performs no loading or execution.""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import Annotated, Literal - -from pydantic import BaseModel, ConfigDict, Field, model_validator - - -class _StrictFrozenModel(BaseModel): - """Immutable schema node that rejects unknown YAML fields.""" - - model_config = ConfigDict(extra="forbid", frozen=True) - - -class HuggingFaceDatasetSource(_StrictFrozenModel): - type: Literal["huggingface_dataset"] - repo_id: str = Field(min_length=1) - revision: str = Field(pattern=r"^[0-9a-fA-F]{40}$") - config: str | None = None - split: str | None = None - allow_patterns: tuple[str, ...] = () - - -class HuggingFaceSpaceSource(_StrictFrozenModel): - type: Literal["huggingface_space"] - repo_id: str = Field(min_length=1) - revision: str = Field(pattern=r"^[0-9a-fA-F]{40}$") - allow_patterns: tuple[str, ...] = () - - -class GitRawSource(_StrictFrozenModel): - type: Literal["git_raw"] - repository: str = Field(min_length=1) - revision: str = Field(pattern=r"^[0-9a-fA-F]{40}$") - path: str = Field(min_length=1) - - -class LocalSource(_StrictFrozenModel): - type: Literal["local"] - path: str = Field(min_length=1) - format: Literal["csv", "json", "jsonl", "parquet"] - sha256: str = Field(pattern=r"^[0-9a-fA-F]{64}$") - - -SourceSpec = Annotated[ - HuggingFaceDatasetSource | HuggingFaceSpaceSource | GitRawSource | LocalSource, - Field(discriminator="type"), -] - - -class DatasetFields(_StrictFrozenModel): - id: str = Field(min_length=1) - instruction: str = Field(min_length=1) - category: str | None = None - - -class DatasetSpec(_StrictFrozenModel): - """Dataset sources, loader adapter, and canonical field mapping.""" - - adapter: str = Field(min_length=1) - sources: dict[str, SourceSpec] = Field(min_length=1) - fields: DatasetFields - - -class SingleTurnGeneration(_StrictFrozenModel): - mode: Literal["single_turn_chat"] - - -class MultiTurnGeneration(_StrictFrozenModel): - mode: Literal["multi_turn_chat"] - category_temperatures: dict[str, float] = Field(default_factory=dict) - - @model_validator(mode="after") - def _validate_temperatures(self) -> MultiTurnGeneration: - invalid = { - category: temperature - for category, temperature in self.category_temperatures.items() - if not category or temperature < 0 - } - if invalid: - raise ValueError( - f"category temperatures require names and non-negative values: {invalid}" - ) - return self - - -class NoBaseline(_StrictFrozenModel): - strategy: Literal["none"] - - -class RuntimeRequiredBaseline(_StrictFrozenModel): - strategy: Literal["runtime_required"] - - -class TaskDefaultBaseline(_StrictFrozenModel): - strategy: Literal["task_default"] - reference_id: str = Field(min_length=1) - allow_runtime_override: bool = True - - -class CategoryDefaultsBaseline(_StrictFrozenModel): - strategy: Literal["category_defaults"] - category_field: str = Field(min_length=1) - references: dict[str, str] = Field(min_length=1) - allow_runtime_override: bool = True - - -class OfficialOutputsBaseline(_StrictFrozenModel): - strategy: Literal["official_outputs"] - source: str = Field(min_length=1) - - -BaselineSpec = Annotated[ - NoBaseline - | RuntimeRequiredBaseline - | TaskDefaultBaseline - | CategoryDefaultsBaseline - | OfficialOutputsBaseline, - Field(discriminator="strategy"), -] - -SwapMode = Literal["fixed", "both"] - - -class PairwiseJudgeSpec(_StrictFrozenModel): - default_prompt: str = Field(min_length=1) - parser: str = Field(min_length=1) - default_swap_mode: SwapMode = "fixed" - allowed_swap_modes: tuple[SwapMode, ...] = ("fixed", "both") - default_temperature: float | None = None - - @model_validator(mode="after") - def _default_must_be_allowed(self) -> PairwiseJudgeSpec: - if not self.allowed_swap_modes: - raise ValueError("allowed_swap_modes must not be empty") - if self.default_swap_mode not in self.allowed_swap_modes: - raise ValueError("default_swap_mode must be present in allowed_swap_modes") - if len(set(self.allowed_swap_modes)) != len(self.allowed_swap_modes): - raise ValueError("allowed_swap_modes must not contain duplicates") - return self - - -class ScoringSpec(_StrictFrozenModel): - adapter: str = Field(min_length=1) - primary_metric: str = Field(min_length=1) - higher_is_better: bool - - -class PairwiseProtocol(_StrictFrozenModel): - """Task-owned generation, baseline, judging, and scoring behavior.""" - - runner: Literal["pairwise"] - generation: SingleTurnGeneration - baseline: BaselineSpec - judge: PairwiseJudgeSpec - scoring: ScoringSpec - - -class MTBenchJudgeSpec(PairwiseJudgeSpec): - turns_mode: Literal["both", "single", "multi"] = "both" - fastchat_prompt_preset: str = Field(min_length=1) - fastchat_temperature: float = Field(ge=0) - reference_categories: tuple[str, ...] = () - - @model_validator(mode="after") - def _validate_reference_categories(self) -> MTBenchJudgeSpec: - if any(not category for category in self.reference_categories): - raise ValueError("reference categories must not be empty") - if len(set(self.reference_categories)) != len(self.reference_categories): - raise ValueError("reference categories must not contain duplicates") - return self - - -class MTBenchProtocol(_StrictFrozenModel): - """Task policy used by the specialized multi-turn MT-Bench runner.""" - - runner: Literal["mt_bench"] - generation: MultiTurnGeneration - baseline: BaselineSpec - judge: MTBenchJudgeSpec - scoring: ScoringSpec - - -ProtocolSpec = Annotated[ - PairwiseProtocol | MTBenchProtocol, - Field(discriminator="runner"), -] - - -class TaskMetadata(_StrictFrozenModel): - reference_implementation: str | None = None - paper: str | None = None - - -class SuffixVariants(_StrictFrozenModel): - """Validated suffixes selecting views of one task definition.""" - - selector: str = Field(pattern=r"^[A-Za-z_][A-Za-z0-9_]*$") - values: tuple[str, ...] = Field(min_length=1) - groups: dict[str, tuple[str, ...]] = Field(default_factory=dict) - - @model_validator(mode="after") - def _validate_variants(self) -> SuffixVariants: - if any(not value for value in self.values): - raise ValueError("variant values must not be empty") - if any(not group for group in self.groups): - raise ValueError("variant group names must not be empty") - if len(set(self.values)) != len(self.values): - raise ValueError("variant values must not contain duplicates") - overlap = set(self.values) & set(self.groups) - if overlap: - raise ValueError(f"variant values and groups overlap: {sorted(overlap)}") - known = set(self.values) - for group, members in self.groups.items(): - if not members: - raise ValueError(f"variant group {group!r} must not be empty") - if len(set(members)) != len(members): - raise ValueError(f"variant group {group!r} must not contain duplicates") - unknown = sorted(set(members) - known) - if unknown: - raise ValueError( - f"variant group {group!r} references unknown values: {unknown}" - ) - return self - - -class TaskSpec(_StrictFrozenModel): - """Complete validated definition of one registered task.""" - - schema_version: Literal[1] - task: str = Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]*$") - task_version: int = Field(ge=1) - description: str = Field(min_length=1) - tags: tuple[str, ...] = () - dataset: DatasetSpec - protocol: ProtocolSpec - variants: SuffixVariants | None = None - metadata: TaskMetadata = Field(default_factory=TaskMetadata) - - @model_validator(mode="after") - def _validate_task(self) -> TaskSpec: - if len(set(self.tags)) != len(self.tags): - raise ValueError("tags must not contain duplicates") - source_names = set(self.dataset.sources) - baseline = self.protocol.baseline - if ( - isinstance(baseline, OfficialOutputsBaseline) - and baseline.source not in source_names - ): - raise ValueError( - f"official baseline source {baseline.source!r} is not declared " - "in dataset.sources" - ) - if isinstance(baseline, CategoryDefaultsBaseline) and ( - self.dataset.fields.category != baseline.category_field - ): - raise ValueError( - "category-default baseline must use dataset.fields.category" - ) - return self - - -@dataclass(frozen=True) -class ResourceDigest: - """Hash of one YAML resource used to construct a task.""" - - path: str - sha256: str - - -@dataclass(frozen=True) -class TaskProvenance: - """Source paths and hashes needed to identify a resolved definition.""" - - source_path: str - source_sha256: str - resolved_sha256: str - resources: tuple[ResourceDigest, ...] - - -@dataclass(frozen=True) -class TaskSelection: - """Runtime selector resolved from a task-family suffix.""" - - selector: str - name: str - values: tuple[str, ...] - - -@dataclass(frozen=True) -class ResolvedTaskSpec: - """Validated task plus the provenance of its resolved YAML.""" - - spec: TaskSpec - provenance: TaskProvenance - invocation_task: str | None = None - selection: TaskSelection | None = None - - @property - def task(self) -> str: - return self.invocation_task or self.spec.task - - @property - def definition_task(self) -> str: - """Task ID written in the source YAML, before suffix selection.""" - return self.spec.task - - def model_dump(self) -> dict[str, object]: - """Return the normalized task definition without provenance.""" - return self.spec.model_dump(mode="json") diff --git a/judgearena/tasks/schema/__init__.py b/judgearena/tasks/schema/__init__.py new file mode 100644 index 0000000..8ca0766 --- /dev/null +++ b/judgearena/tasks/schema/__init__.py @@ -0,0 +1,77 @@ +"""Public schema API for declarative task definitions. + +The implementation is split by responsibility so dataset and evaluation +protocol schemas can evolve independently. Re-exporting the public models here +keeps existing imports stable. +""" + +from judgearena.tasks.schema.baselines import ( + BaselineSpec, + CategoryDefaultsBaseline, + NoBaseline, + OfficialOutputsBaseline, + RuntimeRequiredBaseline, + TaskDefaultBaseline, +) +from judgearena.tasks.schema.dataset import DatasetFields, DatasetSpec +from judgearena.tasks.schema.mt_bench import ( + MTBenchJudgeSpec, + MTBenchProtocol, + MultiTurnGeneration, +) +from judgearena.tasks.schema.pairwise import ( + PairwiseJudgeSpec, + PairwiseProtocol, + ScoringSpec, + SingleTurnGeneration, + SwapMode, +) +from judgearena.tasks.schema.resolved import ( + ResolvedTaskSpec, + ResourceDigest, + TaskProvenance, + TaskSelection, +) +from judgearena.tasks.schema.sources import ( + GitRawSource, + HuggingFaceDatasetSource, + HuggingFaceSpaceSource, + SourceSpec, +) +from judgearena.tasks.schema.task import ( + ProtocolSpec, + SuffixVariants, + TaskMetadata, + TaskSpec, +) + +__all__ = [ + "BaselineSpec", + "CategoryDefaultsBaseline", + "DatasetFields", + "DatasetSpec", + "GitRawSource", + "HuggingFaceDatasetSource", + "HuggingFaceSpaceSource", + "MTBenchJudgeSpec", + "MTBenchProtocol", + "MultiTurnGeneration", + "NoBaseline", + "OfficialOutputsBaseline", + "PairwiseJudgeSpec", + "PairwiseProtocol", + "ProtocolSpec", + "ResolvedTaskSpec", + "ResourceDigest", + "RuntimeRequiredBaseline", + "ScoringSpec", + "SingleTurnGeneration", + "SourceSpec", + "SuffixVariants", + "SwapMode", + "TaskDefaultBaseline", + "TaskMetadata", + "TaskProvenance", + "TaskSelection", + "TaskSpec", +] diff --git a/judgearena/tasks/schema/base.py b/judgearena/tasks/schema/base.py new file mode 100644 index 0000000..b965f1e --- /dev/null +++ b/judgearena/tasks/schema/base.py @@ -0,0 +1,9 @@ +"""Shared behavior for immutable task schema models.""" + +from pydantic import BaseModel, ConfigDict + + +class StrictFrozenModel(BaseModel): + """Immutable schema node that rejects unknown YAML fields.""" + + model_config = ConfigDict(extra="forbid", frozen=True) diff --git a/judgearena/tasks/schema/baselines.py b/judgearena/tasks/schema/baselines.py new file mode 100644 index 0000000..515fc06 --- /dev/null +++ b/judgearena/tasks/schema/baselines.py @@ -0,0 +1,43 @@ +"""Baseline-selection policies shared by pairwise protocols.""" + +from typing import Annotated, Literal + +from pydantic import Field + +from judgearena.tasks.schema.base import StrictFrozenModel + + +class NoBaseline(StrictFrozenModel): + strategy: Literal["none"] + + +class RuntimeRequiredBaseline(StrictFrozenModel): + strategy: Literal["runtime_required"] + + +class TaskDefaultBaseline(StrictFrozenModel): + strategy: Literal["task_default"] + reference_id: str = Field(min_length=1) + allow_runtime_override: bool = True + + +class CategoryDefaultsBaseline(StrictFrozenModel): + strategy: Literal["category_defaults"] + category_field: str = Field(min_length=1) + references: dict[str, str] = Field(min_length=1) + allow_runtime_override: bool = True + + +class OfficialOutputsBaseline(StrictFrozenModel): + strategy: Literal["official_outputs"] + source: str = Field(min_length=1) + + +BaselineSpec = Annotated[ + NoBaseline + | RuntimeRequiredBaseline + | TaskDefaultBaseline + | CategoryDefaultsBaseline + | OfficialOutputsBaseline, + Field(discriminator="strategy"), +] diff --git a/judgearena/tasks/schema/dataset.py b/judgearena/tasks/schema/dataset.py new file mode 100644 index 0000000..73dc573 --- /dev/null +++ b/judgearena/tasks/schema/dataset.py @@ -0,0 +1,22 @@ +"""Common dataset contract for declarative tasks.""" + +from pydantic import Field + +from judgearena.tasks.schema.base import StrictFrozenModel +from judgearena.tasks.schema.sources import SourceSpec + + +class DatasetFields(StrictFrozenModel): + """Map upstream columns to JudgeArena's canonical task fields.""" + + id: str = Field(min_length=1) + instruction: str = Field(min_length=1) + category: str | None = None + + +class DatasetSpec(StrictFrozenModel): + """Dataset sources, loader adapter, and canonical field mapping.""" + + adapter: str = Field(min_length=1) + sources: dict[str, SourceSpec] = Field(min_length=1) + fields: DatasetFields diff --git a/judgearena/tasks/schema/mt_bench.py b/judgearena/tasks/schema/mt_bench.py new file mode 100644 index 0000000..e9b66e9 --- /dev/null +++ b/judgearena/tasks/schema/mt_bench.py @@ -0,0 +1,54 @@ +"""Schema owned by the specialized multi-turn MT-Bench protocol.""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import Field, model_validator + +from judgearena.tasks.schema.base import StrictFrozenModel +from judgearena.tasks.schema.baselines import BaselineSpec +from judgearena.tasks.schema.pairwise import PairwiseJudgeSpec, ScoringSpec + + +class MultiTurnGeneration(StrictFrozenModel): + mode: Literal["multi_turn_chat"] + category_temperatures: dict[str, float] = Field(default_factory=dict) + + @model_validator(mode="after") + def _validate_temperatures(self) -> MultiTurnGeneration: + invalid = { + category: temperature + for category, temperature in self.category_temperatures.items() + if not category or temperature < 0 + } + if invalid: + raise ValueError( + f"category temperatures require names and non-negative values: {invalid}" + ) + return self + + +class MTBenchJudgeSpec(PairwiseJudgeSpec): + turns_mode: Literal["both", "single", "multi"] = "both" + fastchat_prompt_preset: str = Field(min_length=1) + fastchat_temperature: float = Field(ge=0) + reference_categories: tuple[str, ...] = () + + @model_validator(mode="after") + def _validate_reference_categories(self) -> MTBenchJudgeSpec: + if any(not category for category in self.reference_categories): + raise ValueError("reference categories must not be empty") + if len(set(self.reference_categories)) != len(self.reference_categories): + raise ValueError("reference categories must not contain duplicates") + return self + + +class MTBenchProtocol(StrictFrozenModel): + """Task policy used by the specialized multi-turn MT-Bench runner.""" + + runner: Literal["mt_bench"] + generation: MultiTurnGeneration + baseline: BaselineSpec + judge: MTBenchJudgeSpec + scoring: ScoringSpec diff --git a/judgearena/tasks/schema/pairwise.py b/judgearena/tasks/schema/pairwise.py new file mode 100644 index 0000000..b206473 --- /dev/null +++ b/judgearena/tasks/schema/pairwise.py @@ -0,0 +1,48 @@ +"""Schema for the common single-turn pairwise evaluation protocol.""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import Field, model_validator + +from judgearena.tasks.schema.base import StrictFrozenModel +from judgearena.tasks.schema.baselines import BaselineSpec + + +class SingleTurnGeneration(StrictFrozenModel): + mode: Literal["single_turn_chat"] + + +SwapMode = Literal["fixed", "both"] + + +class PairwiseJudgeSpec(StrictFrozenModel): + default_prompt: str = Field(min_length=1) + default_swap_mode: SwapMode = "fixed" + allowed_swap_modes: tuple[SwapMode, ...] = ("fixed", "both") + default_temperature: float | None = None + + @model_validator(mode="after") + def _default_must_be_allowed(self) -> PairwiseJudgeSpec: + if not self.allowed_swap_modes: + raise ValueError("allowed_swap_modes must not be empty") + if self.default_swap_mode not in self.allowed_swap_modes: + raise ValueError("default_swap_mode must be present in allowed_swap_modes") + if len(set(self.allowed_swap_modes)) != len(self.allowed_swap_modes): + raise ValueError("allowed_swap_modes must not contain duplicates") + return self + + +class ScoringSpec(StrictFrozenModel): + adapter: str = Field(min_length=1) + + +class PairwiseProtocol(StrictFrozenModel): + """Task-owned generation, baseline, judging, and scoring behavior.""" + + runner: Literal["pairwise"] + generation: SingleTurnGeneration + baseline: BaselineSpec + judge: PairwiseJudgeSpec + scoring: ScoringSpec diff --git a/judgearena/tasks/schema/resolved.py b/judgearena/tasks/schema/resolved.py new file mode 100644 index 0000000..93702b5 --- /dev/null +++ b/judgearena/tasks/schema/resolved.py @@ -0,0 +1,57 @@ +"""Runtime records produced after loading and resolving task YAML.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from judgearena.tasks.schema.task import TaskSpec + + +@dataclass(frozen=True) +class ResourceDigest: + """Hash of one YAML resource used to construct a task.""" + + path: str + sha256: str + + +@dataclass(frozen=True) +class TaskProvenance: + """Source paths and hashes needed to identify a resolved definition.""" + + source_path: str + source_sha256: str + resolved_sha256: str + resources: tuple[ResourceDigest, ...] + + +@dataclass(frozen=True) +class TaskSelection: + """Runtime selector resolved from a task-family suffix.""" + + selector: str + name: str + values: tuple[str, ...] + + +@dataclass(frozen=True) +class ResolvedTaskSpec: + """Validated task plus the provenance of its resolved YAML.""" + + spec: TaskSpec + provenance: TaskProvenance + invocation_task: str | None = None + selection: TaskSelection | None = None + + @property + def task(self) -> str: + return self.invocation_task or self.spec.task + + @property + def definition_task(self) -> str: + """Task ID written in the source YAML, before suffix selection.""" + return self.spec.task + + def model_dump(self) -> dict[str, object]: + """Return the normalized task definition without provenance.""" + return self.spec.model_dump(mode="json") diff --git a/judgearena/tasks/schema/sources.py b/judgearena/tasks/schema/sources.py new file mode 100644 index 0000000..6bdefad --- /dev/null +++ b/judgearena/tasks/schema/sources.py @@ -0,0 +1,35 @@ +"""Pinned external resource schemas used by task datasets.""" + +from typing import Annotated, Literal + +from pydantic import Field + +from judgearena.tasks.schema.base import StrictFrozenModel + + +class HuggingFaceDatasetSource(StrictFrozenModel): + type: Literal["huggingface_dataset"] + repo_id: str = Field(min_length=1) + revision: str = Field(pattern=r"^[0-9a-fA-F]{40}$") + config: str | None = None + allow_patterns: tuple[str, ...] = () + + +class HuggingFaceSpaceSource(StrictFrozenModel): + type: Literal["huggingface_space"] + repo_id: str = Field(min_length=1) + revision: str = Field(pattern=r"^[0-9a-fA-F]{40}$") + allow_patterns: tuple[str, ...] = () + + +class GitRawSource(StrictFrozenModel): + type: Literal["git_raw"] + repository: str = Field(min_length=1) + revision: str = Field(pattern=r"^[0-9a-fA-F]{40}$") + path: str = Field(min_length=1) + + +SourceSpec = Annotated[ + HuggingFaceDatasetSource | HuggingFaceSpaceSource | GitRawSource, + Field(discriminator="type"), +] diff --git a/judgearena/tasks/schema/task.py b/judgearena/tasks/schema/task.py new file mode 100644 index 0000000..2c9713c --- /dev/null +++ b/judgearena/tasks/schema/task.py @@ -0,0 +1,94 @@ +"""Top-level task definition and cross-section validation.""" + +from __future__ import annotations + +from typing import Annotated, Literal + +from pydantic import Field, model_validator + +from judgearena.tasks.schema.base import StrictFrozenModel +from judgearena.tasks.schema.baselines import ( + CategoryDefaultsBaseline, + OfficialOutputsBaseline, +) +from judgearena.tasks.schema.dataset import DatasetSpec +from judgearena.tasks.schema.mt_bench import MTBenchProtocol +from judgearena.tasks.schema.pairwise import PairwiseProtocol + +ProtocolSpec = Annotated[ + PairwiseProtocol | MTBenchProtocol, + Field(discriminator="runner"), +] + + +class TaskMetadata(StrictFrozenModel): + reference_implementation: str | None = None + paper: str | None = None + + +class SuffixVariants(StrictFrozenModel): + """Validated suffixes selecting views of one task definition.""" + + selector: str = Field(pattern=r"^[A-Za-z_][A-Za-z0-9_]*$") + values: tuple[str, ...] = Field(min_length=1) + groups: dict[str, tuple[str, ...]] = Field(default_factory=dict) + + @model_validator(mode="after") + def _validate_variants(self) -> SuffixVariants: + if any(not value for value in self.values): + raise ValueError("variant values must not be empty") + if any(not group for group in self.groups): + raise ValueError("variant group names must not be empty") + if len(set(self.values)) != len(self.values): + raise ValueError("variant values must not contain duplicates") + overlap = set(self.values) & set(self.groups) + if overlap: + raise ValueError(f"variant values and groups overlap: {sorted(overlap)}") + known = set(self.values) + for group, members in self.groups.items(): + if not members: + raise ValueError(f"variant group {group!r} must not be empty") + if len(set(members)) != len(members): + raise ValueError(f"variant group {group!r} must not contain duplicates") + unknown = sorted(set(members) - known) + if unknown: + raise ValueError( + f"variant group {group!r} references unknown values: {unknown}" + ) + return self + + +class TaskSpec(StrictFrozenModel): + """Complete validated definition of one registered task.""" + + schema_version: Literal[1] + task: str = Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]*$") + task_version: int = Field(ge=1) + description: str = Field(min_length=1) + tags: tuple[str, ...] = () + dataset: DatasetSpec + protocol: ProtocolSpec + variants: SuffixVariants | None = None + metadata: TaskMetadata = Field(default_factory=TaskMetadata) + + @model_validator(mode="after") + def _validate_task(self) -> TaskSpec: + if len(set(self.tags)) != len(self.tags): + raise ValueError("tags must not contain duplicates") + source_names = set(self.dataset.sources) + baseline = self.protocol.baseline + if ( + isinstance(baseline, OfficialOutputsBaseline) + and baseline.source not in source_names + ): + raise ValueError( + f"official baseline source {baseline.source!r} is not declared " + "in dataset.sources" + ) + if isinstance(baseline, CategoryDefaultsBaseline) and ( + self.dataset.fields.category != baseline.category_field + ): + raise ValueError( + "category-default baseline must use dataset.fields.category" + ) + return self diff --git a/tests/test_generate_and_evaluate.py b/tests/test_generate_and_evaluate.py index f650c24..fbd6c4e 100644 --- a/tests/test_generate_and_evaluate.py +++ b/tests/test_generate_and_evaluate.py @@ -6,14 +6,17 @@ import judgearena.benchmarks.execution as benchmark_execution import judgearena.benchmarks.pairwise.runner as generate_and_evaluate import judgearena.benchmarks.registry as benchmark_registry -from judgearena.benchmarks.pairwise.baselines import native_pairwise_baseline -from judgearena.benchmarks.pairwise.runner import ( +import judgearena.benchmarks.runner as benchmark_runner +from judgearena.benchmarks.pairwise.baselines import ( BaselinePlan, - _resolve_baseline_plan, - run_pairwise, + native_pairwise_baseline, + resolve_baseline_plan, ) +from judgearena.benchmarks.pairwise.runner import run_pairwise from judgearena.benchmarks.registry import BenchmarkAdapter, resolve_benchmark_adapter from judgearena.config import RunConfig +from judgearena.datasets.pairwise import PairwiseTaskData +from judgearena.tasks.registry import get_packaged_task def _cfg( @@ -77,9 +80,20 @@ def mock_external_data_and_cache(monkeypatch): monkeypatch.setattr( generate_and_evaluate, - "try_load_dataset_completions", + "_try_load_legacy_dataset_completions", lambda dataset, model, n_instructions: None, ) + monkeypatch.setattr( + generate_and_evaluate, + "load_pairwise_task_data", + lambda task, n_instructions=None: PairwiseTaskData( + instructions=( + instructions.head(n_instructions) + if n_instructions is not None + else instructions + ) + ), + ) def _run_without_cache(fun, **_kwargs): return fun() @@ -97,49 +111,53 @@ def _instructions(ids: list[str], categories: list[str] | None = None) -> pd.Dat def test_resolve_plan_v01_flat_default(): - plan = _resolve_baseline_plan( - task="arena-hard-v0.1", - model_b=None, - instructions_df=_instructions(["q1", "q2"]), + plan = resolve_baseline_plan( + task_id="arena-hard-v0.1", + task=get_packaged_task("arena-hard-v0.1"), + runtime_baseline=None, + instructions=_instructions(["q1", "q2"]), ) - assert plan.is_flat + assert plan.is_single_model assert plan.single_model == "gpt-4-0314" def test_resolve_plan_v20_routes_per_category(): - plan = _resolve_baseline_plan( - task="arena-hard-v2.0", - model_b=None, - instructions_df=_instructions( + plan = resolve_baseline_plan( + task_id="arena-hard-v2.0", + task=get_packaged_task("arena-hard-v2.0"), + runtime_baseline=None, + instructions=_instructions( ["qh", "qc"], categories=["hard_prompt", "creative_writing"], ), ) - assert not plan.is_flat + assert not plan.is_single_model assert plan.baseline_by_index.loc["qh"] == "o3-mini-2025-01-31" assert plan.baseline_by_index.loc["qc"] == "gemini-2.0-flash-001" def test_resolve_plan_alpaca_eval_uses_native_baseline(): - plan = _resolve_baseline_plan( - task="alpaca-eval", - model_b=None, - instructions_df=_instructions(["q1", "q2"]), + plan = resolve_baseline_plan( + task_id="alpaca-eval", + task=get_packaged_task("alpaca-eval"), + runtime_baseline=None, + instructions=_instructions(["q1", "q2"]), ) - assert plan.is_flat + assert plan.is_single_model assert plan.single_model == "gpt4_1106_preview" def test_resolve_plan_explicit_model_b_overrides_native(): - plan = _resolve_baseline_plan( - task="arena-hard-v2.0", - model_b="override", - instructions_df=_instructions( + plan = resolve_baseline_plan( + task_id="arena-hard-v2.0", + task=get_packaged_task("arena-hard-v2.0"), + runtime_baseline="override", + instructions=_instructions( ["q1", "q2"], categories=["hard_prompt", "creative_writing"], ), ) - assert plan.is_flat + assert plan.is_single_model assert plan.single_model == "override" @@ -181,43 +199,66 @@ def test_registered_task_runner_wins_over_legacy_fallback(monkeypatch): assert benchmark_registry.resolve_benchmark_adapter("yaml-task") is pairwise +def test_benchmark_dispatch_passes_the_resolved_task(monkeypatch): + resolved = SimpleNamespace( + spec=SimpleNamespace(protocol=SimpleNamespace(runner="pairwise")) + ) + captured = {} + pairwise = BenchmarkAdapter( + "pairwise", + frozenset(), + lambda cfg, task: captured.update(cfg=cfg, task=task) or "result", + ) + monkeypatch.setattr(benchmark_registry, "benchmark_adapters", lambda: (pairwise,)) + monkeypatch.setattr(benchmark_registry, "get_packaged_task", lambda _task: resolved) + cfg = SimpleNamespace(task="yaml-task") + + result = benchmark_runner.run_benchmark(cfg) + + assert result == "result" + assert captured == {"cfg": cfg, "task": resolved} + + def test_resolve_plan_task_without_native_baseline_requires_model_b(): with pytest.raises(ValueError, match="baseline"): - _resolve_baseline_plan( - task="fluency-french", - model_b=None, - instructions_df=_instructions(["q1"]), + resolve_baseline_plan( + task_id="fluency-french", + task=None, + runtime_baseline=None, + instructions=_instructions(["q1"]), ) def test_resolve_plan_v20_missing_category_raises(): with pytest.raises(ValueError, match="category"): - _resolve_baseline_plan( - task="arena-hard-v2.0", - model_b=None, - instructions_df=_instructions(["q1"]), + resolve_baseline_plan( + task_id="arena-hard-v2.0", + task=get_packaged_task("arena-hard-v2.0"), + runtime_baseline=None, + instructions=_instructions(["q1"]), ) def test_resolve_plan_v20_unknown_category_raises(): with pytest.raises(ValueError, match="brand_new"): - _resolve_baseline_plan( - task="arena-hard-v2.0", - model_b=None, - instructions_df=_instructions(["q1"], categories=["brand_new"]), + resolve_baseline_plan( + task_id="arena-hard-v2.0", + task=get_packaged_task("arena-hard-v2.0"), + runtime_baseline=None, + instructions=_instructions(["q1"], categories=["brand_new"]), ) def test_baseline_plan_flat_repeats_model(): plan = BaselinePlan.flat("b", index=pd.Index(["a", "b"])) - assert plan.is_flat + assert plan.is_single_model assert plan.baseline_by_index.tolist() == ["b", "b"] def test_baseline_plan_per_row_preserves_order(): series = pd.Series(["m1", "m2"], index=["a", "b"], name="model_B") plan = BaselinePlan.per_row(series) - assert not plan.is_flat + assert not plan.is_single_model assert plan.unique_models == ["m1", "m2"] diff --git a/tests/test_instruction_dataset.py b/tests/test_instruction_dataset.py index 521195f..2e57a5c 100644 --- a/tests/test_instruction_dataset.py +++ b/tests/test_instruction_dataset.py @@ -1,13 +1,14 @@ from pathlib import Path +from types import SimpleNamespace import pandas as pd import pytest -import judgearena.benchmarks.pairwise.runner as generate_and_evaluate import judgearena.datasets as instruction_dataset import judgearena.datasets.arena_hard as arena_hard import judgearena.datasets.judgearena_tables as judgearena_tables import judgearena.datasets.m_arenahard as m_arenahard +import judgearena.datasets.pairwise as pairwise_data from judgearena.datasets.arena_hard import ( _build_instructions, _build_model_outputs, @@ -384,7 +385,7 @@ def test_load_instructions_surfaces_category_for_v20(monkeypatch): assert df.loc["q2", "category"] == "creative_writing" -def test_try_load_dataset_completions_uses_dataset_output_file(monkeypatch, tmp_path): +def test_pairwise_task_data_uses_declared_adapter_outputs(monkeypatch, tmp_path): tables_dir = tmp_path / "tables" / "model_outputs" tables_dir.mkdir(parents=True, exist_ok=True) output_path = tables_dir / "arena-hard-v2.0.csv.zip" @@ -396,17 +397,26 @@ def test_try_load_dataset_completions_uses_dataset_output_file(monkeypatch, tmp_ } ).to_csv(output_path, index=False) - monkeypatch.setattr(generate_and_evaluate, "data_root", tmp_path) - monkeypatch.setattr( - arena_hard, - "load_task_model_outputs", - lambda task, path: pd.read_csv(output_path), + adapter = SimpleNamespace( + name="arena_hard", + load_instructions=lambda task, path: pd.DataFrame( + { + "instruction_index": [0, 1], + "instruction": ["q0", "q1"], + } + ), + load_model_outputs=lambda task, path: pd.read_csv(output_path), ) + monkeypatch.setattr(pairwise_data, "resolve_dataset_adapter", lambda name: adapter) + task = get_packaged_task("arena-hard-v2.0") + assert task is not None - loaded = generate_and_evaluate.try_load_dataset_completions( - dataset="arena-hard-v2.0", model="baseline", n_instructions=None + task_data = pairwise_data.load_pairwise_task_data( + task, local_tables_path=tmp_path / "tables" ) + loaded = task_data.model_completion("baseline") + assert loaded is not None - assert loaded["completion"].tolist() == ["b0", "b1"] - assert loaded["instruction_index"].tolist() == [0, 1] + assert loaded.tolist() == ["b0", "b1"] + assert loaded.index.tolist() == [0, 1] diff --git a/tests/test_mt_bench_downloads.py b/tests/test_mt_bench_downloads.py index b142245..26f5eeb 100644 --- a/tests/test_mt_bench_downloads.py +++ b/tests/test_mt_bench_downloads.py @@ -199,6 +199,7 @@ def fake_generate_multiturn(**kwargs): completions_a, completions_b = mt_bench_runner._generate_mt_bench_completions( cfg=cfg, + protocol=get_packaged_task("mt-bench").spec.protocol, questions_df=questions_df, ) @@ -236,6 +237,7 @@ def test_generate_mt_bench_completions_reports_missing_baseline_rows(monkeypatch with pytest.raises(ValueError, match="missing 1 question"): mt_bench_runner._generate_mt_bench_completions( cfg=cfg, + protocol=get_packaged_task("mt-bench").spec.protocol, questions_df=questions_df, ) @@ -299,7 +301,7 @@ def test_run_mt_bench_resolves_native_baseline_and_judge_controls( monkeypatch.setattr( mt_bench_runner, "_generate_mt_bench_completions", - lambda cfg, questions_df: ( + lambda cfg, protocol, questions_df: ( pd.DataFrame( {"completion_turn_1": ["A1"], "completion_turn_2": ["A2"]}, index=questions_df.index, @@ -343,13 +345,13 @@ def fake_run_mt_bench_fastchat(**kwargs): run={"result_folder": str(tmp_path)}, ) - mt_bench_runner.run_mt_bench_benchmark(cfg) + mt_bench_runner.run_mt_bench_benchmark(cfg, get_packaged_task("mt-bench")) assert cfg.model.baseline == "gpt-4" assert captured["make_model"]["max_model_len"] == 65536 assert captured["make_model"]["tensor_parallel_size"] == 4 assert captured["fastchat"]["cfg"].generation.truncate_judge_input_chars == 80000 - assert captured["fastchat"]["fastchat_prompt_preset"] == "default" + assert captured["fastchat"]["protocol"].judge.fastchat_prompt_preset == "default" assert captured["fastchat"]["resolved_prompt"].preset_name == ( FASTCHAT_PAIRWISE_PROMPT_PRESET ) @@ -370,7 +372,7 @@ def test_run_mt_bench_defaults_to_delegated_fastchat(monkeypatch, tmp_path): monkeypatch.setattr( mt_bench_runner, "_generate_mt_bench_completions", - lambda cfg, questions_df: ( + lambda cfg, protocol, questions_df: ( pd.DataFrame( {"completion_turn_1": ["A1"], "completion_turn_2": ["A2"]}, index=questions_df.index, @@ -411,11 +413,11 @@ def fake_run_mt_bench_fastchat(**kwargs): run={"result_folder": str(tmp_path)}, ) - mt_bench_runner.run_mt_bench_benchmark(cfg) + mt_bench_runner.run_mt_bench_benchmark(cfg, get_packaged_task("mt-bench")) assert cfg.model.baseline == "gpt-4" assert captured["make_model"]["temperature"] == 0.0 - assert captured["fastchat"]["fastchat_prompt_preset"] == "default" + assert captured["fastchat"]["protocol"].judge.fastchat_prompt_preset == "default" assert captured["fastchat"]["resolved_prompt"].preset_name == ( FASTCHAT_PAIRWISE_PROMPT_PRESET ) @@ -435,7 +437,7 @@ def test_run_mt_bench_concrete_prompt_preset_uses_preset_judging(monkeypatch, tm monkeypatch.setattr( mt_bench_runner, "_generate_mt_bench_completions", - lambda cfg, questions_df: ( + lambda cfg, protocol, questions_df: ( pd.DataFrame( {"completion_turn_1": ["A1"], "completion_turn_2": ["A2"]}, index=questions_df.index, @@ -476,7 +478,7 @@ def fake_run_mt_bench_preset(**kwargs): run={"result_folder": str(tmp_path)}, ) - mt_bench_runner.run_mt_bench_benchmark(cfg) + mt_bench_runner.run_mt_bench_benchmark(cfg, get_packaged_task("mt-bench")) assert captured["preset"]["resolved_prompt"].preset_name == ( "default_with_explanation" @@ -529,6 +531,7 @@ def fake_generate_multiturn(**kwargs): mt_bench_runner._generate_mt_bench_completions( cfg=cfg, + protocol=get_packaged_task("mt-bench").spec.protocol, questions_df=questions_df, ) @@ -558,7 +561,7 @@ def test_run_mt_bench_forwards_strip_thinking_to_fastchat_judge(monkeypatch, tmp monkeypatch.setattr( mt_bench_runner, "_generate_mt_bench_completions", - lambda cfg, questions_df: ( + lambda cfg, protocol, questions_df: ( pd.DataFrame( {"completion_turn_1": ["A1"], "completion_turn_2": ["A2"]}, index=questions_df.index, @@ -588,7 +591,7 @@ def fake_judge(**kwargs): run={"result_folder": str(tmp_path)}, ) - mt_bench_runner.run_mt_bench_benchmark(cfg) + mt_bench_runner.run_mt_bench_benchmark(cfg, get_packaged_task("mt-bench")) assert captured["judge"]["strip_thinking_before_judging"] is True assert captured["judge"]["reference_categories"] == ( diff --git a/tests/test_pairwise_scoring.py b/tests/test_pairwise_scoring.py new file mode 100644 index 0000000..cb2ca4e --- /dev/null +++ b/tests/test_pairwise_scoring.py @@ -0,0 +1,17 @@ +"""Tests for runtime pairwise scoring adapters.""" + +import pandas as pd +import pytest + +from judgearena.benchmarks.pairwise.scoring import PAIRWISE_SCORERS + + +def test_pairwise_win_rate_scorer_owns_metric_semantics(): + scorer = PAIRWISE_SCORERS["pairwise_win_rate"] + + summary = scorer.summarize(pd.Series([0.0, 1.0, 0.5])) + + assert scorer.primary_metric == "winrate" + assert scorer.higher_is_better is True + assert summary.num_battles == 3 + assert summary.winrate == pytest.approx(0.5) diff --git a/tests/test_task_registry.py b/tests/test_task_registry.py index 00eac89..13d49e2 100644 --- a/tests/test_task_registry.py +++ b/tests/test_task_registry.py @@ -41,15 +41,10 @@ def _task_definition(task: str = "test-task") -> dict[str, object]: }, "judge": { "default_prompt": "default", - "parser": "pairwise_preference", "default_swap_mode": "fixed", "allowed_swap_modes": ["fixed", "both"], }, - "scoring": { - "adapter": "pairwise_win_rate", - "primary_metric": "winrate", - "higher_is_better": True, - }, + "scoring": {"adapter": "pairwise_win_rate"}, }, } @@ -136,7 +131,7 @@ def test_packaged_registry_discovers_versioned_tasks(): assert mt_bench.spec.dataset.sources["benchmark"].revision == ( "a4b674ca573c24143824ac7f60d9173e7081e37d" ) - assert alpaca.spec.protocol.scoring.primary_metric == "winrate" + assert alpaca.spec.protocol.scoring.adapter == "pairwise_win_rate" def test_find_returns_none_for_unregistered_task(): @@ -339,6 +334,20 @@ def test_registry_rejects_unknown_adapter_id(tmp_path): load_tasks(tmp_path) +def test_registry_rejects_unknown_scorer_id(tmp_path): + definition = _task_definition() + definition["protocol"]["scoring"]["adapter"] = "missing_scorer" + _write_family( + tmp_path, + family="example", + filename="test-task.yaml", + definition=definition, + ) + + with pytest.raises(TaskDefinitionError, match="unknown scorer"): + load_tasks(tmp_path) + + def test_official_outputs_must_reference_declared_source(tmp_path): definition = _task_definition() definition["protocol"]["baseline"] = {