Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ multi-objective Pareto optimization, and Bayesian model stacking.

The core pattern (from [MFAI §4–5](https://github.com/jcm-sci/trade-study)):

1. **Model world** — Generate synthetic ground truth with known latent state
1. **Simulate** — Generate synthetic ground truth with known latent state
2. **Observe** — Apply a realistic observation model (noise, masks, bias)
3. **Score** — Evaluate structured observables against *known truth* using
proper scoring rules (not noisy observations)
Expand All @@ -29,7 +29,7 @@ benchmark), where each phase filters configs for the next.

```
trade_study/
├── protocols.py ModelWorld, Scorer, Observable, Annotation, ResultsTable
├── protocols.py Simulator, Scorer, Observable, Annotation, ResultsTable
├── design.py Factor screening (SALib) and grid construction (pyDOE3)
├── scoring.py Proper scoring rules (scoringrules) and calibration
├── pareto.py Non-dominated sorting and indicators (pymoo)
Expand Down Expand Up @@ -83,7 +83,7 @@ grid = build_grid(factors, method="lhs", n_samples=500)

# Run hierarchical study
study = Study(
world=MyModelWorld(),
world=MySimulator(),
scorer=MyScorer(),
observables=[coverage, rank_err, wall_time],
phases=[
Expand Down
4 changes: 2 additions & 2 deletions src/trade_study/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@
from .protocols import (
Annotation,
Direction,
ModelWorld,
Observable,
ResultsTable,
Scorer,
Simulator,
Tier,
TrialResult,
)
Expand All @@ -19,11 +19,11 @@
__all__ = [
"Annotation",
"Direction",
"ModelWorld",
"Observable",
"Phase",
"ResultsTable",
"Scorer",
"Simulator",
"Study",
"Tier",
"TrialResult",
Expand Down
12 changes: 6 additions & 6 deletions src/trade_study/protocols.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,10 @@ class Observable:


@runtime_checkable
class ModelWorld(Protocol):
class Simulator(Protocol):
"""Protocol for generating ground truth and observations.

A model world produces (truth, observations) pairs where truth is the
A simulator produces (truth, observations) pairs where truth is the
known latent state and observations are what a real system would see.
"""

Expand Down Expand Up @@ -76,8 +76,8 @@ def score(
"""Score a single trial, returning values for each observable.

Args:
truth: Known latent state from the model world.
observations: Observed data from the model world.
truth: Known latent state from the simulator.
observations: Observed data from the simulator.
config: The configuration that produced this trial.

Returns:
Expand All @@ -88,7 +88,7 @@ def score(

@dataclass
class TrialResult:
"""Result of a single model world trial."""
"""Result of a single simulation trial."""

config: dict[str, Any]
scores: dict[str, float]
Expand All @@ -100,7 +100,7 @@ class TrialResult:
class Annotation:
"""External information attached to configurations.

Used for costs, constraints, or metadata not computed by the model world
Used for costs, constraints, or metadata not computed by the simulator
(e.g. dollar costs from a surveillance costing sheet).

Attributes:
Expand Down
12 changes: 6 additions & 6 deletions src/trade_study/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@
from .protocols import (
Annotation,
Direction,
ModelWorld,
Observable,
ResultsTable,
Scorer,
Simulator,
TrialResult,
)

Expand All @@ -28,7 +28,7 @@


def _run_single(
world: ModelWorld,
world: Simulator,
scorer: Scorer,
config: dict[str, Any],
) -> TrialResult:
Expand All @@ -45,7 +45,7 @@ def _run_single(


def run_grid(
world: ModelWorld,
world: Simulator,
scorer: Scorer,
grid: list[dict[str, Any]],
observables: list[Observable],
Expand All @@ -56,7 +56,7 @@ def run_grid(
"""Run all configurations in a grid.

Args:
world: Model world that generates (truth, observations).
world: Simulator that generates (truth, observations).
scorer: Scorer that evaluates observables.
grid: List of config dicts to evaluate.
observables: Observable definitions (for column ordering).
Expand Down Expand Up @@ -99,7 +99,7 @@ def run_grid(


def run_adaptive(
world: ModelWorld,
world: Simulator,
scorer: Scorer,
factors: list[Factor],
observables: list[Observable],
Expand All @@ -110,7 +110,7 @@ def run_adaptive(
"""Run adaptive multi-objective optimization via optuna.

Args:
world: Model world.
world: Simulator.
scorer: Scorer for observables.
factors: Factor definitions (from design module).
observables: Observable definitions.
Expand Down
6 changes: 3 additions & 3 deletions src/trade_study/study.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,10 @@

from .protocols import (
Annotation,
ModelWorld,
Observable,
ResultsTable,
Scorer,
Simulator,
)


Expand Down Expand Up @@ -89,15 +89,15 @@ class Study:
"""Multi-phase model criticism study.

Attributes:
world: Model world generating (truth, observations).
world: Simulator generating (truth, observations).
scorer: Scorer evaluating observables against truth.
observables: Observable definitions.
phases: Ordered list of study phases.
annotations: External information (costs, constraints).
factors: Factor definitions (needed for adaptive mode).
"""

world: ModelWorld
world: Simulator
scorer: Scorer
observables: list[Observable]
phases: list[Phase]
Expand Down
Loading