From 4431c9713907ca069bb5c9d9ba299e8c832bfda4 Mon Sep 17 00:00:00 2001 From: RAJVEER42 Date: Fri, 3 Jul 2026 07:27:38 +0530 Subject: [PATCH 1/4] fix(beam): inline R2 to drop the sklearn runtime dependency (#805) openstef-beam imported sklearn.metrics.r2_score in metrics_deterministic but never declared scikit-learn as a dependency, so r2 failed whenever sklearn was absent. Reimplement R2 in numpy, matching scikit-learn for the weighted, constant-target, and fewer-than-two-sample (NaN) cases, and add unit tests covering them. Signed-off-by: RAJVEER42 --- .../metrics/metrics_deterministic.py | 25 ++++++++++++++--- .../metrics/test_metrics_deterministic.py | 28 +++++++++++++++++++ 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/packages/openstef-beam/src/openstef_beam/metrics/metrics_deterministic.py b/packages/openstef-beam/src/openstef_beam/metrics/metrics_deterministic.py index c6ca30af9..925f80823 100644 --- a/packages/openstef-beam/src/openstef_beam/metrics/metrics_deterministic.py +++ b/packages/openstef-beam/src/openstef_beam/metrics/metrics_deterministic.py @@ -19,13 +19,15 @@ import numpy as np import numpy.typing as npt -from sklearn.metrics import r2_score from openstef_core.types import Quantile _Q_05 = Quantile(0.05) _Q_95 = Quantile(0.95) +# R² is undefined for fewer than this many samples (variance needs at least two points). +_MIN_R2_SAMPLES = 2 + def completeness( y: npt.NDArray[np.floating], @@ -527,7 +529,8 @@ def r2( Returns: The R² score as a float. Best possible score is 1.0, and it can be negative (because the model can be arbitrarily worse). A constant model that always - predicts the mean of y_true would get an R² score of 0.0. + predicts the mean of y_true would get an R² score of 0.0. Fewer than two + samples returns NaN, since R² is undefined there (matching scikit-learn). Example: Basic usage with energy load data @@ -552,10 +555,24 @@ def r2( >>> isinstance(score, float) True """ - if len(y_true) == 0 or len(y_pred) == 0: + # R² is undefined for fewer than two samples; match scikit-learn, which + # returns NaN (with a warning) in that case. + if len(y_true) < _MIN_R2_SAMPLES or len(y_pred) < _MIN_R2_SAMPLES: return float("NaN") - return float(r2_score(y_true, y_pred, sample_weight=sample_weights)) + y_true = np.asarray(y_true, dtype=float) + y_pred = np.asarray(y_pred, dtype=float) + weights = np.ones_like(y_true) if sample_weights is None else np.asarray(sample_weights, dtype=float) + + weighted_mean = np.average(y_true, weights=weights) + residual_sum = float(np.sum(weights * (y_true - y_pred) ** 2)) + total_sum = float(np.sum(weights * (y_true - weighted_mean) ** 2)) + + # Constant y_true: match scikit-learn (1.0 for a perfect fit, else 0.0). + if total_sum == 0.0: + return 1.0 if residual_sum == 0.0 else 0.0 + + return float(1.0 - residual_sum / total_sum) def pinball_loss( diff --git a/packages/openstef-beam/tests/unit/metrics/test_metrics_deterministic.py b/packages/openstef-beam/tests/unit/metrics/test_metrics_deterministic.py index d73f069af..4028bc87f 100644 --- a/packages/openstef-beam/tests/unit/metrics/test_metrics_deterministic.py +++ b/packages/openstef-beam/tests/unit/metrics/test_metrics_deterministic.py @@ -15,6 +15,7 @@ mape, pinball_loss, precision_recall, + r2, relative_pinball_loss, riqd, rmae, @@ -624,3 +625,30 @@ def test_pinball_loss_various( # Assert assert abs(result - expected) < 1e-8, f"Expected {expected} but got {result}" + + +def test_r2_perfect_and_constant_predictor(): + """R² is 1.0 for a perfect fit and 0.0 for a constant mean predictor.""" + y_true = np.array([1.0, 2.0, 3.0, 4.0]) + + assert r2(y_true, y_true) == pytest.approx(1.0) + assert r2(y_true, np.full_like(y_true, y_true.mean())) == pytest.approx(0.0) + + +def test_r2_matches_known_values(): + """R² matches hand-computed values, including a sample-weighted case.""" + # 1 - 1.5 / 29.1875 + assert r2(np.array([3.0, -0.5, 2.0, 7.0]), np.array([2.5, 0.0, 2.0, 8.0])) == pytest.approx(0.948608, abs=1e-5) + # Weighted: weighted mean 2.25, residual sum 2.0, total sum 2.75 -> 1 - 2 / 2.75 + weighted = r2( + np.array([1.0, 2.0, 3.0]), + np.array([1.0, 2.0, 4.0]), + sample_weights=np.array([1.0, 1.0, 2.0]), + ) + assert weighted == pytest.approx(0.272727, abs=1e-5) + + +def test_r2_undefined_for_fewer_than_two_samples(): + """Fewer than two samples returns NaN (R² undefined, matches scikit-learn).""" + assert np.isnan(r2(np.array([]), np.array([]))) + assert np.isnan(r2(np.array([5.0]), np.array([5.0]))) From 1751e41c7c60be5ff74b81f9450e34319ed1cb32 Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Thu, 30 Jul 2026 15:08:31 +0200 Subject: [PATCH 2/4] feature: Added smooth handling of edge cases for R2 calculation. Signed-off-by: Egor Dmitriev Signed-off-by: Egor Dmitriev --- .../metrics/metrics_deterministic.py | 28 +++++++++++++++---- .../metrics/test_metrics_deterministic.py | 7 +++++ 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/packages/openstef-beam/src/openstef_beam/metrics/metrics_deterministic.py b/packages/openstef-beam/src/openstef_beam/metrics/metrics_deterministic.py index 925f80823..e92e38878 100644 --- a/packages/openstef-beam/src/openstef_beam/metrics/metrics_deterministic.py +++ b/packages/openstef-beam/src/openstef_beam/metrics/metrics_deterministic.py @@ -25,9 +25,6 @@ _Q_05 = Quantile(0.05) _Q_95 = Quantile(0.95) -# R² is undefined for fewer than this many samples (variance needs at least two points). -_MIN_R2_SAMPLES = 2 - def completeness( y: npt.NDArray[np.floating], @@ -507,7 +504,7 @@ def riqd( return float(riqd) -def r2( +def r2( # noqa: PLR0911 y_true: npt.NDArray[np.floating], y_pred: npt.NDArray[np.floating], *, @@ -555,19 +552,38 @@ def r2( >>> isinstance(score, float) True """ + # This metric currently supports one-dimensional series only. + if y_true.ndim != 1 or y_pred.ndim != 1: + return float("nan") + + if y_true.shape != y_pred.shape: + return float("nan") + # R² is undefined for fewer than two samples; match scikit-learn, which # returns NaN (with a warning) in that case. - if len(y_true) < _MIN_R2_SAMPLES or len(y_pred) < _MIN_R2_SAMPLES: + if len(y_true) < 2 or len(y_pred) < 2: # noqa: PLR2004 (constants would be less readable) return float("NaN") y_true = np.asarray(y_true, dtype=float) y_pred = np.asarray(y_pred, dtype=float) - weights = np.ones_like(y_true) if sample_weights is None else np.asarray(sample_weights, dtype=float) + if sample_weights is None: + weights = np.ones_like(y_true) + else: + weights = np.asarray(sample_weights, dtype=float) + if weights.shape != y_true.shape: + return float("nan") + + weight_sum = np.sum(weights) + if not np.isfinite(weight_sum) or weight_sum == 0.0: + return float("nan") weighted_mean = np.average(y_true, weights=weights) residual_sum = float(np.sum(weights * (y_true - y_pred) ** 2)) total_sum = float(np.sum(weights * (y_true - weighted_mean) ** 2)) + if not np.isfinite(residual_sum) or not np.isfinite(total_sum): + return float("nan") + # Constant y_true: match scikit-learn (1.0 for a perfect fit, else 0.0). if total_sum == 0.0: return 1.0 if residual_sum == 0.0 else 0.0 diff --git a/packages/openstef-beam/tests/unit/metrics/test_metrics_deterministic.py b/packages/openstef-beam/tests/unit/metrics/test_metrics_deterministic.py index 4028bc87f..9bb1e72e6 100644 --- a/packages/openstef-beam/tests/unit/metrics/test_metrics_deterministic.py +++ b/packages/openstef-beam/tests/unit/metrics/test_metrics_deterministic.py @@ -652,3 +652,10 @@ def test_r2_undefined_for_fewer_than_two_samples(): """Fewer than two samples returns NaN (R² undefined, matches scikit-learn).""" assert np.isnan(r2(np.array([]), np.array([]))) assert np.isnan(r2(np.array([5.0]), np.array([5.0]))) + + +def test_r2_constant_target(): + y_true = np.array([5.0, 5.0, 5.0]) + + assert r2(y_true, y_true) == pytest.approx(1.0) + assert r2(y_true, np.array([5.0, 5.0, 6.0])) == pytest.approx(0.0) From f4dfaa568124198d9d640472857db2e885f971fd Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Thu, 30 Jul 2026 15:16:27 +0200 Subject: [PATCH 3/4] feature: Elaborated a gitignore Signed-off-by: Egor Dmitriev Signed-off-by: Egor Dmitriev --- .../src/openstef_beam/metrics/metrics_deterministic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/openstef-beam/src/openstef_beam/metrics/metrics_deterministic.py b/packages/openstef-beam/src/openstef_beam/metrics/metrics_deterministic.py index e92e38878..532309c3d 100644 --- a/packages/openstef-beam/src/openstef_beam/metrics/metrics_deterministic.py +++ b/packages/openstef-beam/src/openstef_beam/metrics/metrics_deterministic.py @@ -504,7 +504,7 @@ def riqd( return float(riqd) -def r2( # noqa: PLR0911 +def r2( # noqa: PLR0911 (too many return statements - intentional to handle edge cases like scikit-learn) y_true: npt.NDArray[np.floating], y_pred: npt.NDArray[np.floating], *, From 9f63478a55fbb9586506678ee507dc234446425b Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Thu, 30 Jul 2026 15:36:17 +0200 Subject: [PATCH 4/4] feature: data error returns nan. logic / structure error throws an error. Signed-off-by: Egor Dmitriev Signed-off-by: Egor Dmitriev --- .../metrics/metrics_deterministic.py | 74 +++++++++++++------ 1 file changed, 51 insertions(+), 23 deletions(-) diff --git a/packages/openstef-beam/src/openstef_beam/metrics/metrics_deterministic.py b/packages/openstef-beam/src/openstef_beam/metrics/metrics_deterministic.py index 532309c3d..33040074d 100644 --- a/packages/openstef-beam/src/openstef_beam/metrics/metrics_deterministic.py +++ b/packages/openstef-beam/src/openstef_beam/metrics/metrics_deterministic.py @@ -504,7 +504,7 @@ def riqd( return float(riqd) -def r2( # noqa: PLR0911 (too many return statements - intentional to handle edge cases like scikit-learn) +def r2( y_true: npt.NDArray[np.floating], y_pred: npt.NDArray[np.floating], *, @@ -517,17 +517,30 @@ def r2( # noqa: PLR0911 (too many return statements - intentional to handle edg well observed outcomes are replicated by the model, based on the proportion of total variation of outcomes explained by the model. + Structurally invalid inputs, such as incompatible shapes or unsupported + dimensions, raise ValueError. Statistically unusable data, such as fewer + than two samples or non-finite values, returns NaN. + Args: y_true: Ground truth values with shape (num_samples,). y_pred: Predicted values with shape (num_samples,). - sample_weights: Optional weights for each sample with shape (num_samples,). - If None, all samples are weighted equally. + sample_weights: Optional weights for each sample with shape + (num_samples,). If None, all samples are weighted equally. Returns: - The R² score as a float. Best possible score is 1.0, and it can be negative - (because the model can be arbitrarily worse). A constant model that always - predicts the mean of y_true would get an R² score of 0.0. Fewer than two - samples returns NaN, since R² is undefined there (matching scikit-learn). + The R² score as a float. The best possible score is 1.0, and the score + can be negative because a model can perform arbitrarily worse than a + constant mean predictor. Fewer than two samples or non-finite data + returns NaN. + + For a constant target, this function follows scikit-learn's default + finite behavior: a perfect prediction returns 1.0 and an imperfect + prediction returns 0.0. + + Raises: + ValueError: If the inputs are not one-dimensional, if y_true and y_pred + have different shapes, or if sample_weights does not have the same + one-dimensional shape as y_true. Example: Basic usage with energy load data @@ -552,43 +565,58 @@ def r2( # noqa: PLR0911 (too many return statements - intentional to handle edg >>> isinstance(score, float) True """ - # This metric currently supports one-dimensional series only. + y_true = np.asarray(y_true, dtype=float) + y_pred = np.asarray(y_pred, dtype=float) + + # Structural contract violations should fail loudly. In particular, + # rejecting dimensions explicitly prevents unintended NumPy broadcasting. if y_true.ndim != 1 or y_pred.ndim != 1: - return float("nan") + error_msg = f"y_true and y_pred must be one-dimensional; got shapes {y_true.shape} and {y_pred.shape}" + raise ValueError(error_msg) if y_true.shape != y_pred.shape: - return float("nan") + error_msg = f"y_true and y_pred must have the same shape; got {y_true.shape} and {y_pred.shape}" + raise ValueError(error_msg) - # R² is undefined for fewer than two samples; match scikit-learn, which - # returns NaN (with a warning) in that case. - if len(y_true) < 2 or len(y_pred) < 2: # noqa: PLR2004 (constants would be less readable) - return float("NaN") - - y_true = np.asarray(y_true, dtype=float) - y_pred = np.asarray(y_pred, dtype=float) if sample_weights is None: weights = np.ones_like(y_true) else: weights = np.asarray(sample_weights, dtype=float) - if weights.shape != y_true.shape: - return float("nan") - weight_sum = np.sum(weights) + if weights.ndim != 1 or weights.shape != y_true.shape: + error_msg = ( + f"sample_weights must be one-dimensional and have the same " + f"shape as y_true; got {weights.shape} and {y_true.shape}" + ) + raise ValueError(error_msg) + + # R² is statistically undefined with fewer than two observations. + if y_true.size < 2: # noqa: PLR2004 (hardcoded constant is too trivial to warrant a named constant) + return float("nan") + + # Non-finite values represent unusable metric data rather than a structural + # programming error. Propagate that condition as an undefined metric. + if not (np.all(np.isfinite(y_true)) and np.all(np.isfinite(y_pred)) and np.all(np.isfinite(weights))): + return float("nan") + + weight_sum = float(np.sum(weights)) if not np.isfinite(weight_sum) or weight_sum == 0.0: return float("nan") - weighted_mean = np.average(y_true, weights=weights) + weighted_mean = float(np.average(y_true, weights=weights)) residual_sum = float(np.sum(weights * (y_true - y_pred) ** 2)) total_sum = float(np.sum(weights * (y_true - weighted_mean) ** 2)) if not np.isfinite(residual_sum) or not np.isfinite(total_sum): return float("nan") - # Constant y_true: match scikit-learn (1.0 for a perfect fit, else 0.0). + # Match scikit-learn's default force_finite=True behavior for a constant + # target: perfect predictions score 1.0, otherwise the score is 0.0. if total_sum == 0.0: return 1.0 if residual_sum == 0.0 else 0.0 - return float(1.0 - residual_sum / total_sum) + score = 1.0 - residual_sum / total_sum + return float(score) if np.isfinite(score) else float("nan") def pinball_loss(