Export result tables to pandas with optional trial metadata

CHANGELOG.md

@@ -6,6 +6,8 @@ All notable changes to this project will be documented in this file.
 
 ### Added
 
+- `ResultsTable.to_dataframe()` exports factor values, scores, annotations and optional flattened trial metadata; pandas remains optional via the `dataframe` extra (#85).
+
 - Replicated trials: `run_grid(..., n_reps=N)` evaluates each design point N times; simulators may opt in to per-replicate randomness via an optional `rep` keyword on `Simulator.generate` (detected by introspection). `ResultsTable.aggregate_replicates()` collapses replicate rows back to per-design-point means with `n_reps`/`score_std` metadata. `Phase.n_reps` forwards this into `Study`, and phase filtering now runs against aggregated design points rather than raw replicates when `n_reps>1` (#112).
 - Surrogate accuracy reporting: `fit_surrogate()`/`SurrogateModel` (and `fit_regime_surrogate()`/`RegimeSurrogate` via passthrough) now compute a uniform k-fold cross-validated `cv_r2`/`cv_rmse` per observable for both the `gp` and `rf` backends. Warns (`warn_below_r2`, default threshold `0.0`) at fit time and in `RegimeSurrogate.recommend()` when an observable's accuracy is too low to trust (#114).
 - `sensitivity_from_table()`: post-hoc Sobol/Morris sensitivity from an already-collected `ResultsTable`, by fitting a cheap surrogate over it (`fit_surrogate`) and running `screen()`'s existing machinery against the surrogate instead of a fresh simulator. Unlike a marginal Spearman correlation, this correctly detects non-monotonic (e.g. U-shaped) factor effects. Returns a `TableSensitivity` with `importance` indices and the surrogate's `surrogate_cv_r2` (#114) so callers can judge whether to trust the result (#113).

README.md

@@ -143,6 +143,7 @@ pip install trade-study[design,pareto]
 | `stacking` | [arviz](https://github.com/arviz-devs/arviz), scipy                                                            | Bayesian and score-based ensemble weights   |
 | `adaptive` | [optuna](https://optuna.org/)                                                                                  | Adaptive multi-objective search (NSGA-II)   |
 | `parallel` | joblib                                                                                                         | Parallel grid execution                     |
+| `dataframe` | pandas | ResultsTable export for analysis and CSV |
 | `all`      | All of the above                                                                                               |                                             |
 
 **Core dependency**: numpy only.
@@ -203,6 +204,8 @@ from trade_study import save_results, load_results
 
 save_results(results, "study_results")
 results = load_results("study_results")
+frame = results.to_dataframe()  # optional pandas extra; includes trial metadata
+frame.to_csv("study_results.csv", index=False)
 ```
 
 ### Multi-fidelity search and surrogates

docs/api/protocols.md

@@ -17,3 +17,15 @@ Core types and interfaces for trade-study workflows.
 ::: trade_study.TrialResult
 
 ::: trade_study.ResultsTable
+
+## DataFrame export
+
+Install `trade-study[dataframe]` and call `results.to_dataframe()` to get one
+row per trial with factor, observable, annotation, and metadata columns.
+Metadata is flattened under `meta.` so replicate counts and uncertainty remain
+available, including `meta.standard_error.<observable>` and raw adaptive means
+in `meta.scores.<observable>`. Use `include_metadata=False` for only factors,
+scores, and annotations. Conflicting column names raise an error instead of
+silently overwriting values. Use pandas `frame.to_csv(path, index=False)` to
+export CSV. Call `aggregate_replicates()` explicitly first if you want grid
+results collapsed to design points.

pyproject.toml

@@ -48,6 +48,9 @@ Changelog = "https://github.com/jcm-sci/trade-study/blob/main/CHANGELOG.md"
 Issues = "https://github.com/jcm-sci/trade-study/issues"
 
 [project.optional-dependencies]
+dataframe = [
+  "pandas>=2.0",
+]
 scoring = [
   "scoringrules>=0.8",
 ]
@@ -77,7 +80,7 @@ surrogate = [
   "scikit-learn>=1.3",
 ]
 all = [
-  "trade-study[scoring,pareto,stacking,design,adaptive,parallel,viz,surrogate]",
+  "trade-study[scoring,pareto,stacking,design,adaptive,parallel,viz,surrogate,dataframe]",
 ]
 examples = [
   "scikit-learn>=1.3",

src/trade_study/protocols.py

@@ -13,6 +13,7 @@
     from collections.abc import Callable
 
     import numpy as np
+    import pandas as pd  # type: ignore[import-untyped]
     from numpy.typing import NDArray
 
 
@@ -288,6 +289,59 @@ class ResultsTable:
     annotation_names: list[str] = field(default_factory=list)
     metadata: list[dict[str, Any]] = field(default_factory=list)
 
+    def to_dataframe(self, *, include_metadata: bool = True) -> pd.DataFrame:
+        """Export one row per trial to a pandas DataFrame.
+
+        Factor columns appear in config insertion order, followed by
+        observables and annotations. Missing config keys become missing
+        values; categorical values retain their original values. Metadata
+        is flattened under ``meta.`` (for example ``meta.rep`` and
+        ``meta.standard_error.loss``), retaining raw adaptive means and
+        replicate uncertainty. This method does not aggregate replicates.
+
+        Args:
+            include_metadata: Include trial metadata, default True.
+
+        Returns:
+            A new DataFrame independent of the score and annotation arrays.
+
+        Raises:
+            ImportError: If pandas is unavailable; install the dataframe extra.
+            ValueError: If column names collide or metadata row counts differ.
+        """
+        try:
+            import pandas as pd
+        except ImportError as error:
+            msg = "DataFrame export requires pandas; install trade-study[dataframe]"
+            raise ImportError(msg) from error
+
+        index = pd.RangeIndex(len(self.configs))
+        frames = [
+            pd.DataFrame(self.configs, index=index),
+            pd.DataFrame(
+                self.scores.copy(), index=index, columns=self.observable_names
+            ),
+        ]
+        if self.annotations is not None:
+            frames.append(
+                pd.DataFrame(
+                    self.annotations.copy(), index=index, columns=self.annotation_names
+                )
+            )
+        if include_metadata and self.metadata:
+            if len(self.metadata) != len(self.configs):
+                msg = "metadata must have one entry per trial"
+                raise ValueError(msg)
+            frames.append(pd.json_normalize(self.metadata).add_prefix("meta."))
+        columns = [column for frame in frames for column in frame.columns]
+        if len(columns) != len(set(columns)):
+            msg = (
+                "DataFrame column names collide; rename conflicting factors, "
+                "observables or annotations"
+            )
+            raise ValueError(msg)
+        return pd.concat(frames, axis=1)
+
     def feasible(self, constraints: list[Constraint]) -> NDArray[np.bool_]:
         """Return a boolean mask indicating which rows satisfy all constraints.
 

tests/test_dataframe.py

@@ -0,0 +1,110 @@
+"""DataFrame exports preserve results and statistical metadata (#85)."""
+
+from __future__ import annotations
+
+import builtins
+from typing import TYPE_CHECKING, Any
+
+import numpy as np
+import pytest
+
+from trade_study import ResultsTable
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping, Sequence
+
+
+def _table() -> ResultsTable:
+    return ResultsTable(
+        configs=[{"method": "a", "x": 1}, {"method": "b", "x": 2}],
+        scores=np.array([[0.1], [np.nan]]),
+        observable_names=["loss"],
+        annotations=np.array([[10.0], [20.0]]),
+        annotation_names=["cost"],
+        metadata=[
+            {
+                "rep": 0,
+                "design_point": 0,
+                "standard_error": {"loss": 0.01},
+                "n_reps": {"loss": 3},
+                "scores": {"loss": 0.05},
+            },
+            {"rep": 1, "design_point": 0},
+        ],
+    )
+
+
+def test_export_preserves_order_values_and_nested_metadata() -> None:
+    table = _table()
+    frame = table.to_dataframe()
+    assert list(frame.columns[:4]) == ["method", "x", "loss", "cost"]
+    assert frame["method"].tolist() == ["a", "b"]
+    assert frame["meta.rep"].tolist() == [0, 1]
+    assert frame.loc[0, "meta.standard_error.loss"] == pytest.approx(0.01)
+    assert frame.loc[0, "meta.n_reps.loss"] == 3
+    assert frame.loc[0, "meta.scores.loss"] == pytest.approx(0.05)
+    assert np.isnan(frame.loc[1, "loss"])
+    frame.loc[0, "loss"] = 9.0
+    assert table.scores[0, 0] == pytest.approx(0.1)
+    frame.loc[0, "cost"] = 99.0
+    assert table.annotations is not None
+    assert table.annotations[0, 0] == pytest.approx(10.0)
+
+
+def test_metadata_can_be_omitted() -> None:
+    assert list(_table().to_dataframe(include_metadata=False).columns) == [
+        "method",
+        "x",
+        "loss",
+        "cost",
+    ]
+
+
+def test_sparse_configs_and_empty_tables() -> None:
+    table = ResultsTable(
+        configs=[{"a": "one"}, {"b": 2}],
+        scores=np.array([[1.0], [2.0]]),
+        observable_names=["loss"],
+    )
+    frame = table.to_dataframe()
+    assert list(frame.columns) == ["a", "b", "loss"]
+    assert frame.loc[0, "a"] == "one"
+    assert np.isnan(frame.loc[0, "b"])
+    empty = ResultsTable(configs=[], scores=np.empty((0, 1)), observable_names=["loss"])
+    assert list(empty.to_dataframe().columns) == ["loss"]
+    assert empty.to_dataframe().empty
+
+
+@pytest.mark.parametrize("name", ["loss", "cost", "meta.rep"])
+def test_column_collisions_raise(name: str) -> None:
+    table = _table()
+    table.configs[0][name] = 1
+    with pytest.raises(ValueError, match="collide"):
+        table.to_dataframe()
+
+
+def test_metadata_length_is_validated() -> None:
+    table = _table()
+    table.metadata = [{}]
+    with pytest.raises(ValueError, match="one entry per trial"):
+        table.to_dataframe()
+
+
+def test_missing_pandas_has_install_guidance(monkeypatch: pytest.MonkeyPatch) -> None:
+    original = builtins.__import__
+
+    def unavailable(
+        name: str,
+        globals_dict: Mapping[str, Any] | None = None,
+        locals_dict: Mapping[str, Any] | None = None,
+        fromlist: Sequence[str] = (),
+        level: int = 0,
+    ) -> Any:
+        if name == "pandas":
+            msg = "pandas intentionally unavailable"
+            raise ImportError(msg)
+        return original(name, globals_dict, locals_dict, fromlist, level)
+
+    monkeypatch.setattr(builtins, "__import__", unavailable)
+    with pytest.raises(ImportError, match=r"trade-study\[dataframe\]"):
+        _table().to_dataframe()