diff --git a/CHANGELOG.md b/CHANGELOG.md index e4e632c..b7d7f9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - `random_state` constructor kwarg on `VBPCA`: seeds parameter initialization and any auto-generated xprobe mask (`int`, `np.random.Generator`, or `None`, following the sklearn convention). Surfaced via `get_params()`/`set_params()`/`get_options()` (#109). +### Fixed +- `select_n_components()` now respects a caller-supplied `xprobe_fraction` when auto-generating a held-out probe set for the `"prms"`/`"cost"` selection metrics. Previously `_ensure_metric_opts` ignored it entirely and always used a hardcoded 10% probe fraction, regardless of what `xprobe_fraction` (e.g. from `recommend_config()`) was passed in — silently training every model-selection candidate on less data than the caller configured (#122). + ### Changed - **Behavior change:** the default (`random_state=None`) now draws fresh entropy on every `fit()` call. Previously, default initialization was silently seeded with a fixed value regardless of configuration, so repeated fits produced identical results without any way to request a different draw. Pass `random_state=` for reproducible runs (#109). - `recommend_config()`'s `missingness` parameter now warns (`UserWarning`) when passed anything other than the default `"auto"`, instead of silently ignoring it. Recommendations are still bucketed by `p` only — the Option A trade study's example recommendations are too sparse per (p-bucket, missingness) cell (23 points across 3 p-buckets x 4 missingness categories) to bucket on responsibly without shipping unreplicated values (#110, see also #111). diff --git a/src/vbpca_py/model_selection.py b/src/vbpca_py/model_selection.py index cfdbe0d..a3fe4fb 100644 --- a/src/vbpca_py/model_selection.py +++ b/src/vbpca_py/model_selection.py @@ -273,11 +273,12 @@ def _ensure_metric_opts( # noqa: PLR0914 Mutates *fit_opts* in place: * **cfstop** — always enabled so the cost learning-curve is populated. - * **xprobe** — when the metric is ``"prms"`` and no probe set has been - supplied, a random 10 % hold-out of observed entries is created. - The corresponding entries are set to NaN in *x_arr* (dense) or - removed from the CSR structure (sparse) so the main fit never sees - them. + * **xprobe** — when no probe set has been supplied, a random hold-out + of observed entries is created, sized by *fit_opts*'s own + ``xprobe_fraction`` when that's set to a positive value, falling + back to a default 10 % otherwise. The corresponding entries are set + to NaN in *x_arr* (dense) or removed from the CSR structure + (sparse) so the main fit never sees them. """ # --- cost: ensure cfstop is non-empty ----------------------------------- cfstop_raw = fit_opts.get("cfstop") @@ -288,11 +289,16 @@ def _ensure_metric_opts( # noqa: PLR0914 if fit_opts.get("xprobe") is not None: return # user already supplied a probe set + configured_fraction = _to_float(fit_opts.get("xprobe_fraction")) + probe_fraction = ( + configured_fraction if configured_fraction > 0.0 else _PROBE_FRACTION + ) + rng = np.random.default_rng(seed) if sp.issparse(x_arr): x_csr = sp.csr_matrix(x_arr) - n_probe = max(1, round(x_csr.nnz * _PROBE_FRACTION)) + n_probe = max(1, round(x_csr.nnz * probe_fraction)) probe_idx = rng.choice(x_csr.nnz, size=n_probe, replace=False) # Build xprobe as a copy, then zero-out non-probe in probe @@ -321,7 +327,7 @@ def _ensure_metric_opts( # noqa: PLR0914 obs_mask = ~np.isnan(x_dense) obs_rows, obs_cols = np.nonzero(obs_mask) - n_probe = max(1, round(len(obs_rows) * _PROBE_FRACTION)) + n_probe = max(1, round(len(obs_rows) * probe_fraction)) probe_idx = rng.choice(len(obs_rows), size=n_probe, replace=False) probe_rows: np.ndarray = obs_rows[probe_idx] diff --git a/tests/test_model_selection.py b/tests/test_model_selection.py index 008de82..0d57c94 100644 --- a/tests/test_model_selection.py +++ b/tests/test_model_selection.py @@ -95,6 +95,32 @@ def test_select_n_components_rejects_invalid_metric() -> None: select_n_components(x, config=cfg) +def test_ensure_metric_opts_respects_configured_xprobe_fraction() -> None: + """A caller-supplied xprobe_fraction sizes the auto-generated probe set (#122).""" + rng = np.random.default_rng(0) + x = rng.standard_normal((100, 80)) + fit_opts: dict[str, object] = {"xprobe_fraction": 0.02} + + ms._ensure_metric_opts(fit_opts, x.copy(), None, SelectionConfig(metric="prms")) + + xprobe = np.asarray(fit_opts["xprobe"]) + n_probe = int(np.sum(~np.isnan(xprobe))) + assert n_probe == pytest.approx(x.size * 0.02, rel=0.1) + + +def test_ensure_metric_opts_falls_back_to_default_probe_fraction() -> None: + """No xprobe_fraction configured -> the historical 10% default applies (#122).""" + rng = np.random.default_rng(0) + x = rng.standard_normal((100, 80)) + fit_opts: dict[str, object] = {} + + ms._ensure_metric_opts(fit_opts, x.copy(), None, SelectionConfig(metric="prms")) + + xprobe = np.asarray(fit_opts["xprobe"]) + n_probe = int(np.sum(~np.isnan(xprobe))) + assert n_probe == pytest.approx(x.size * ms._PROBE_FRACTION, rel=0.1) + + def test_select_n_components_normalizes_component_candidates() -> None: rng = np.random.default_rng(4) x = _low_rank_data(rng, n_features=5, n_samples=7, rank=2)