From c6a729d5d4e4e937f7a13e1a106f7667e6a0d6ed Mon Sep 17 00:00:00 2001 From: "Joshua C. Macdonald" Date: Fri, 2 Oct 2026 21:51:34 -0400 Subject: [PATCH] feat: export result tables with statistical metadata (#85) --- CHANGELOG.md | 2 + README.md | 3 + docs/api/protocols.md | 12 ++++ pyproject.toml | 5 +- src/trade_study/protocols.py | 54 +++++++++++++++++ tests/test_dataframe.py | 110 +++++++++++++++++++++++++++++++++++ 6 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 tests/test_dataframe.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e92539..1281fd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ All notable changes to this project will be documented in this file. ### Added +- `ResultsTable.to_dataframe()` exports factor values, scores, annotations and optional flattened trial metadata; pandas remains optional via the `dataframe` extra (#85). + - Replicated trials: `run_grid(..., n_reps=N)` evaluates each design point N times; simulators may opt in to per-replicate randomness via an optional `rep` keyword on `Simulator.generate` (detected by introspection). `ResultsTable.aggregate_replicates()` collapses replicate rows back to per-design-point means with `n_reps`/`score_std` metadata. `Phase.n_reps` forwards this into `Study`, and phase filtering now runs against aggregated design points rather than raw replicates when `n_reps>1` (#112). - Surrogate accuracy reporting: `fit_surrogate()`/`SurrogateModel` (and `fit_regime_surrogate()`/`RegimeSurrogate` via passthrough) now compute a uniform k-fold cross-validated `cv_r2`/`cv_rmse` per observable for both the `gp` and `rf` backends. Warns (`warn_below_r2`, default threshold `0.0`) at fit time and in `RegimeSurrogate.recommend()` when an observable's accuracy is too low to trust (#114). - `sensitivity_from_table()`: post-hoc Sobol/Morris sensitivity from an already-collected `ResultsTable`, by fitting a cheap surrogate over it (`fit_surrogate`) and running `screen()`'s existing machinery against the surrogate instead of a fresh simulator. Unlike a marginal Spearman correlation, this correctly detects non-monotonic (e.g. U-shaped) factor effects. Returns a `TableSensitivity` with `importance` indices and the surrogate's `surrogate_cv_r2` (#114) so callers can judge whether to trust the result (#113). diff --git a/README.md b/README.md index 65893f0..ce83449 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,7 @@ pip install trade-study[design,pareto] | `stacking` | [arviz](https://github.com/arviz-devs/arviz), scipy | Bayesian and score-based ensemble weights | | `adaptive` | [optuna](https://optuna.org/) | Adaptive multi-objective search (NSGA-II) | | `parallel` | joblib | Parallel grid execution | +| `dataframe` | pandas | ResultsTable export for analysis and CSV | | `all` | All of the above | | **Core dependency**: numpy only. @@ -203,6 +204,8 @@ from trade_study import save_results, load_results save_results(results, "study_results") results = load_results("study_results") +frame = results.to_dataframe() # optional pandas extra; includes trial metadata +frame.to_csv("study_results.csv", index=False) ``` ### Multi-fidelity search and surrogates diff --git a/docs/api/protocols.md b/docs/api/protocols.md index 8c37777..77b6f37 100644 --- a/docs/api/protocols.md +++ b/docs/api/protocols.md @@ -17,3 +17,15 @@ Core types and interfaces for trade-study workflows. ::: trade_study.TrialResult ::: trade_study.ResultsTable + +## DataFrame export + +Install `trade-study[dataframe]` and call `results.to_dataframe()` to get one +row per trial with factor, observable, annotation, and metadata columns. +Metadata is flattened under `meta.` so replicate counts and uncertainty remain +available, including `meta.standard_error.` and raw adaptive means +in `meta.scores.`. Use `include_metadata=False` for only factors, +scores, and annotations. Conflicting column names raise an error instead of +silently overwriting values. Use pandas `frame.to_csv(path, index=False)` to +export CSV. Call `aggregate_replicates()` explicitly first if you want grid +results collapsed to design points. diff --git a/pyproject.toml b/pyproject.toml index 55d2ea6..c88404f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,9 @@ Changelog = "https://github.com/jcm-sci/trade-study/blob/main/CHANGELOG.md" Issues = "https://github.com/jcm-sci/trade-study/issues" [project.optional-dependencies] +dataframe = [ + "pandas>=2.0", +] scoring = [ "scoringrules>=0.8", ] @@ -77,7 +80,7 @@ surrogate = [ "scikit-learn>=1.3", ] all = [ - "trade-study[scoring,pareto,stacking,design,adaptive,parallel,viz,surrogate]", + "trade-study[scoring,pareto,stacking,design,adaptive,parallel,viz,surrogate,dataframe]", ] examples = [ "scikit-learn>=1.3", diff --git a/src/trade_study/protocols.py b/src/trade_study/protocols.py index d60c0c5..4d5984f 100644 --- a/src/trade_study/protocols.py +++ b/src/trade_study/protocols.py @@ -13,6 +13,7 @@ from collections.abc import Callable import numpy as np + import pandas as pd # type: ignore[import-untyped] from numpy.typing import NDArray @@ -288,6 +289,59 @@ class ResultsTable: annotation_names: list[str] = field(default_factory=list) metadata: list[dict[str, Any]] = field(default_factory=list) + def to_dataframe(self, *, include_metadata: bool = True) -> pd.DataFrame: + """Export one row per trial to a pandas DataFrame. + + Factor columns appear in config insertion order, followed by + observables and annotations. Missing config keys become missing + values; categorical values retain their original values. Metadata + is flattened under ``meta.`` (for example ``meta.rep`` and + ``meta.standard_error.loss``), retaining raw adaptive means and + replicate uncertainty. This method does not aggregate replicates. + + Args: + include_metadata: Include trial metadata, default True. + + Returns: + A new DataFrame independent of the score and annotation arrays. + + Raises: + ImportError: If pandas is unavailable; install the dataframe extra. + ValueError: If column names collide or metadata row counts differ. + """ + try: + import pandas as pd + except ImportError as error: + msg = "DataFrame export requires pandas; install trade-study[dataframe]" + raise ImportError(msg) from error + + index = pd.RangeIndex(len(self.configs)) + frames = [ + pd.DataFrame(self.configs, index=index), + pd.DataFrame( + self.scores.copy(), index=index, columns=self.observable_names + ), + ] + if self.annotations is not None: + frames.append( + pd.DataFrame( + self.annotations.copy(), index=index, columns=self.annotation_names + ) + ) + if include_metadata and self.metadata: + if len(self.metadata) != len(self.configs): + msg = "metadata must have one entry per trial" + raise ValueError(msg) + frames.append(pd.json_normalize(self.metadata).add_prefix("meta.")) + columns = [column for frame in frames for column in frame.columns] + if len(columns) != len(set(columns)): + msg = ( + "DataFrame column names collide; rename conflicting factors, " + "observables or annotations" + ) + raise ValueError(msg) + return pd.concat(frames, axis=1) + def feasible(self, constraints: list[Constraint]) -> NDArray[np.bool_]: """Return a boolean mask indicating which rows satisfy all constraints. diff --git a/tests/test_dataframe.py b/tests/test_dataframe.py new file mode 100644 index 0000000..3a6d34b --- /dev/null +++ b/tests/test_dataframe.py @@ -0,0 +1,110 @@ +"""DataFrame exports preserve results and statistical metadata (#85).""" + +from __future__ import annotations + +import builtins +from typing import TYPE_CHECKING, Any + +import numpy as np +import pytest + +from trade_study import ResultsTable + +if TYPE_CHECKING: + from collections.abc import Mapping, Sequence + + +def _table() -> ResultsTable: + return ResultsTable( + configs=[{"method": "a", "x": 1}, {"method": "b", "x": 2}], + scores=np.array([[0.1], [np.nan]]), + observable_names=["loss"], + annotations=np.array([[10.0], [20.0]]), + annotation_names=["cost"], + metadata=[ + { + "rep": 0, + "design_point": 0, + "standard_error": {"loss": 0.01}, + "n_reps": {"loss": 3}, + "scores": {"loss": 0.05}, + }, + {"rep": 1, "design_point": 0}, + ], + ) + + +def test_export_preserves_order_values_and_nested_metadata() -> None: + table = _table() + frame = table.to_dataframe() + assert list(frame.columns[:4]) == ["method", "x", "loss", "cost"] + assert frame["method"].tolist() == ["a", "b"] + assert frame["meta.rep"].tolist() == [0, 1] + assert frame.loc[0, "meta.standard_error.loss"] == pytest.approx(0.01) + assert frame.loc[0, "meta.n_reps.loss"] == 3 + assert frame.loc[0, "meta.scores.loss"] == pytest.approx(0.05) + assert np.isnan(frame.loc[1, "loss"]) + frame.loc[0, "loss"] = 9.0 + assert table.scores[0, 0] == pytest.approx(0.1) + frame.loc[0, "cost"] = 99.0 + assert table.annotations is not None + assert table.annotations[0, 0] == pytest.approx(10.0) + + +def test_metadata_can_be_omitted() -> None: + assert list(_table().to_dataframe(include_metadata=False).columns) == [ + "method", + "x", + "loss", + "cost", + ] + + +def test_sparse_configs_and_empty_tables() -> None: + table = ResultsTable( + configs=[{"a": "one"}, {"b": 2}], + scores=np.array([[1.0], [2.0]]), + observable_names=["loss"], + ) + frame = table.to_dataframe() + assert list(frame.columns) == ["a", "b", "loss"] + assert frame.loc[0, "a"] == "one" + assert np.isnan(frame.loc[0, "b"]) + empty = ResultsTable(configs=[], scores=np.empty((0, 1)), observable_names=["loss"]) + assert list(empty.to_dataframe().columns) == ["loss"] + assert empty.to_dataframe().empty + + +@pytest.mark.parametrize("name", ["loss", "cost", "meta.rep"]) +def test_column_collisions_raise(name: str) -> None: + table = _table() + table.configs[0][name] = 1 + with pytest.raises(ValueError, match="collide"): + table.to_dataframe() + + +def test_metadata_length_is_validated() -> None: + table = _table() + table.metadata = [{}] + with pytest.raises(ValueError, match="one entry per trial"): + table.to_dataframe() + + +def test_missing_pandas_has_install_guidance(monkeypatch: pytest.MonkeyPatch) -> None: + original = builtins.__import__ + + def unavailable( + name: str, + globals_dict: Mapping[str, Any] | None = None, + locals_dict: Mapping[str, Any] | None = None, + fromlist: Sequence[str] = (), + level: int = 0, + ) -> Any: + if name == "pandas": + msg = "pandas intentionally unavailable" + raise ImportError(msg) + return original(name, globals_dict, locals_dict, fromlist, level) + + monkeypatch.setattr(builtins, "__import__", unavailable) + with pytest.raises(ImportError, match=r"trade-study\[dataframe\]"): + _table().to_dataframe()