From 8d511c4435665fb681b18e114c89d683547fa2d1 Mon Sep 17 00:00:00 2001 From: rogueslasher Date: Tue, 14 Apr 2026 16:52:43 +0530 Subject: [PATCH 01/12] feat: add class_names support to Precision and Recall metrics --- ignite/metrics/precision.py | 21 +++++++ tests/ignite/metrics/test_precision.py | 81 ++++++++++++++++++++++++++ 2 files changed, 102 insertions(+) diff --git a/ignite/metrics/precision.py b/ignite/metrics/precision.py index be33b268b15e..69cc84fb991b 100644 --- a/ignite/metrics/precision.py +++ b/ignite/metrics/precision.py @@ -23,7 +23,17 @@ def __init__( is_multilabel: bool = False, device: str | torch.device = torch.device("cpu"), skip_unrolling: bool = False, + class_names: list[str] | None = None, ): + if class_names is not None: + if not isinstance(class_names, (list, tuple)) or not all(isinstance(n, str) for n in class_names): + raise ValueError("class_names must be a list of strings") + if average is not False and average is not None: + raise ValueError( + f"class_names is only applicable when average = False or None,but got average={average!r}" + ) + self._class_names = class_names + if not (average is None or isinstance(average, bool) or average in ["macro", "micro", "weighted", "samples"]): raise ValueError( "Argument average should be None or a boolean or one of values" @@ -157,6 +167,13 @@ def compute(self) -> torch.Tensor | float: elif self._average == "macro": return cast(torch.Tensor, fraction).mean().item() else: + if self._class_names is not None: + if len(self._class_names) != fraction.shape[0]: + raise ValueError( + f"class_names has {len(self._class_names)} entries but the metric computed " + f"{fraction.shape[0]} classes." + ) + return dict(zip(self._class_names, fraction.tolist())) return fraction @@ -246,6 +263,10 @@ class Precision(_BasePrecisionRecall): skip_unrolling: specifies whether output should be unrolled before being fed to update method. Should be true for multi-output model, for example, if ``y_pred`` contains multi-output as ``(y_pred_a, y_pred_b)`` Alternatively, ``output_transform`` can be used to handle this. + class_names: list of class name strings used to label per-class output when ``average=False`` + or ``average=None``. If provided, :meth:`compute` returns a ``dict`` mapping each class + name to its metric value instead of a tensor. Must match the number of classes inferred + from the data. Default: ``None``. Examples: diff --git a/tests/ignite/metrics/test_precision.py b/tests/ignite/metrics/test_precision.py index 7bd80540f5de..3b4283ff757c 100644 --- a/tests/ignite/metrics/test_precision.py +++ b/tests/ignite/metrics/test_precision.py @@ -325,6 +325,87 @@ def test_incorrect_y_classes(average): assert pr._updated is False +@pytest.mark.parametrize( + "invalid_class_names", + [ + [0, 1, 2], # list of ints + "cat", # string instead of list + [0.1, 0.2], # list of floats + ["cat", 1], # mixed + ], +) +def test_class_names_invalid_type(invalid_class_names): + with pytest.raises(ValueError, match="class_names must be a list of strings"): + Precision(average=False, class_names=invalid_class_names) + + +@pytest.mark.parametrize("average", ["macro", "micro", "weighted", "samples", True]) +def test_class_names_incompatible_average(average): + with pytest.raises(ValueError, match="class_names is only applicable when average=False or average=None"): + Precision(average=average, class_names=["cat", "dog", "horse"]) + + +@pytest.mark.parametrize("average", [False, None]) +def test_class_names_multiclass(average): + class_names = ["cat", "dog", "horse"] + pr = Precision(average=average, class_names=class_names) + + y_pred = torch.tensor( + [ + [0.0266, 0.1719, 0.3055], + [0.6886, 0.3978, 0.8176], + [0.9230, 0.0197, 0.8395], + [0.1785, 0.2670, 0.6084], + [0.8448, 0.7177, 0.7288], + ] + ) + y = torch.tensor([2, 0, 2, 1, 0]) + + pr.update((y_pred, y)) + result = pr.compute() + + assert isinstance(result, dict) + assert list(result.keys()) == class_names + assert result == pytest.approx({"cat": 0.5, "dog": 0.0, "horse": 0.3333333333333333}) + + +def test_class_names_length_mismatch(): + pr = Precision(average=False, class_names=["cat", "dog"]) + + y_pred = torch.tensor( + [ + [0.0266, 0.1719, 0.3055], + [0.6886, 0.3978, 0.8176], + [0.9230, 0.0197, 0.8395], + ] + ) + y = torch.tensor([2, 0, 1]) + + pr.update((y_pred, y)) + with pytest.raises(ValueError, match="class_names has 2 entries but the metric computed 3 classes"): + pr.compute() + + +def test_class_names_none_returns_tensor(): + pr = Precision(average=False) + + y_pred = torch.tensor( + [ + [0.0266, 0.1719, 0.3055], + [0.6886, 0.3978, 0.8176], + [0.9230, 0.0197, 0.8395], + [0.1785, 0.2670, 0.6084], + [0.8448, 0.7177, 0.7288], + ] + ) + y = torch.tensor([2, 0, 2, 1, 0]) + + pr.update((y_pred, y)) + result = pr.compute() + + assert isinstance(result, torch.Tensor) + + @pytest.mark.usefixtures("distributed") class TestDistributed: @pytest.mark.parametrize("average", [False, "macro", "weighted", "micro"]) From 9185199d0c9729bf88dbea87d0f1f72982374dfc Mon Sep 17 00:00:00 2001 From: rogueslasher Date: Thu, 16 Apr 2026 17:29:47 +0530 Subject: [PATCH 02/12] feedback: Validate class_names length in update() instead of compute() --- ignite/metrics/precision.py | 13 ++++++------- ignite/metrics/recall.py | 10 +++++++++- tests/ignite/metrics/test_precision.py | 3 +-- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/ignite/metrics/precision.py b/ignite/metrics/precision.py index 69cc84fb991b..db2669fe32b7 100644 --- a/ignite/metrics/precision.py +++ b/ignite/metrics/precision.py @@ -30,7 +30,7 @@ def __init__( raise ValueError("class_names must be a list of strings") if average is not False and average is not None: raise ValueError( - f"class_names is only applicable when average = False or None,but got average={average!r}" + f"class_names is only applicable when average=False or average=None, got average={average!r}." ) self._class_names = class_names @@ -168,11 +168,6 @@ def compute(self) -> torch.Tensor | float: return cast(torch.Tensor, fraction).mean().item() else: if self._class_names is not None: - if len(self._class_names) != fraction.shape[0]: - raise ValueError( - f"class_names has {len(self._class_names)} entries but the metric computed " - f"{fraction.shape[0]} classes." - ) return dict(zip(self._class_names, fraction.tolist())) return fraction @@ -449,5 +444,9 @@ def update(self, output: Sequence[torch.Tensor]) -> None: if self._average == "weighted": self._weight += y.sum(dim=0) - + if self._class_names is not None and len(self._class_names) != self._numerator.shape[0]: + raise ValueError( + f"class_names has {len(self._class_names)} entries but the metric computed " + f"{self._numerator.shape[0]} classes." + ) self._updated = True diff --git a/ignite/metrics/recall.py b/ignite/metrics/recall.py index 34da36ced48f..d4d12e6fc39c 100644 --- a/ignite/metrics/recall.py +++ b/ignite/metrics/recall.py @@ -97,7 +97,10 @@ class Recall(_BasePrecisionRecall): skip_unrolling: specifies whether output should be unrolled before being fed to update method. Should be true for multi-output model, for example, if ``y_pred`` contains multi-output as ``(y_pred_a, y_pred_b)`` Alternatively, ``output_transform`` can be used to handle this. - + class_names: list of class name strings used to label per-class output when ``average=False`` + or ``average=None``. If provided, :meth:`compute` returns a ``dict`` mapping each class + name to its metric value instead of a tensor. Must match the number of classes inferred + from the data. Default: ``None``. Examples: For more information on how metric works with :class:`~ignite.engine.engine.Engine`, visit :ref:`attach-engine`. @@ -240,5 +243,10 @@ def update(self, output: Sequence[torch.Tensor]) -> None: if self._average == "weighted": self._weight += y.sum(dim=0) + if self._class_names is not None and len(self._class_names) != self._numerator.shape[0]: + raise ValueError( + f"class_names has {len(self._class_names)} entries but the metric computed " + f"{self._numerator.shape[0]} classes." + ) self._updated = True diff --git a/tests/ignite/metrics/test_precision.py b/tests/ignite/metrics/test_precision.py index 3b4283ff757c..d2121858c6a3 100644 --- a/tests/ignite/metrics/test_precision.py +++ b/tests/ignite/metrics/test_precision.py @@ -381,9 +381,8 @@ def test_class_names_length_mismatch(): ) y = torch.tensor([2, 0, 1]) - pr.update((y_pred, y)) with pytest.raises(ValueError, match="class_names has 2 entries but the metric computed 3 classes"): - pr.compute() + pr.update((y_pred, y)) def test_class_names_none_returns_tensor(): From 372021f323bed9a750d1a8bb5ed80196bebb454f Mon Sep 17 00:00:00 2001 From: rogueslasher Date: Sat, 27 Jun 2026 17:20:11 +0530 Subject: [PATCH 03/12] feat: add class_names support to Fbeta metric --- ignite/metrics/fbeta.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ignite/metrics/fbeta.py b/ignite/metrics/fbeta.py index f8c165b77903..b4cd78c8293d 100644 --- a/ignite/metrics/fbeta.py +++ b/ignite/metrics/fbeta.py @@ -177,6 +177,16 @@ def thresholded_output_transform(output): elif recall._average: raise ValueError("Input recall metric should have average=False") + if precision._class_names is not None: + + def _fbeta_with_class_names(p: dict, r: dict) -> dict: + p_vals = torch.tensor(list(p.values())) + r_vals = torch.tensor(list(r.values())) + scores = (1.0 + beta**2) * p_vals * r_vals / (beta**2 * p_vals + r_vals + 1e-15) + return dict(zip(precision._class_names, scores.tolist())) + + return MetricsLambda(_fbeta_with_class_names, precision, recall) + fbeta = (1.0 + beta**2) * precision * recall / (beta**2 * precision + recall + 1e-15) if average: From 1cb1e2d4d87adcf75735a5376f99616afdf1e1aa Mon Sep 17 00:00:00 2001 From: rogueslasher Date: Tue, 14 Apr 2026 16:52:43 +0530 Subject: [PATCH 04/12] feat: add class_names support to Precision and Recall metrics --- ignite/metrics/precision.py | 21 +++++++ tests/ignite/metrics/test_precision.py | 81 ++++++++++++++++++++++++++ 2 files changed, 102 insertions(+) diff --git a/ignite/metrics/precision.py b/ignite/metrics/precision.py index be33b268b15e..69cc84fb991b 100644 --- a/ignite/metrics/precision.py +++ b/ignite/metrics/precision.py @@ -23,7 +23,17 @@ def __init__( is_multilabel: bool = False, device: str | torch.device = torch.device("cpu"), skip_unrolling: bool = False, + class_names: list[str] | None = None, ): + if class_names is not None: + if not isinstance(class_names, (list, tuple)) or not all(isinstance(n, str) for n in class_names): + raise ValueError("class_names must be a list of strings") + if average is not False and average is not None: + raise ValueError( + f"class_names is only applicable when average = False or None,but got average={average!r}" + ) + self._class_names = class_names + if not (average is None or isinstance(average, bool) or average in ["macro", "micro", "weighted", "samples"]): raise ValueError( "Argument average should be None or a boolean or one of values" @@ -157,6 +167,13 @@ def compute(self) -> torch.Tensor | float: elif self._average == "macro": return cast(torch.Tensor, fraction).mean().item() else: + if self._class_names is not None: + if len(self._class_names) != fraction.shape[0]: + raise ValueError( + f"class_names has {len(self._class_names)} entries but the metric computed " + f"{fraction.shape[0]} classes." + ) + return dict(zip(self._class_names, fraction.tolist())) return fraction @@ -246,6 +263,10 @@ class Precision(_BasePrecisionRecall): skip_unrolling: specifies whether output should be unrolled before being fed to update method. Should be true for multi-output model, for example, if ``y_pred`` contains multi-output as ``(y_pred_a, y_pred_b)`` Alternatively, ``output_transform`` can be used to handle this. + class_names: list of class name strings used to label per-class output when ``average=False`` + or ``average=None``. If provided, :meth:`compute` returns a ``dict`` mapping each class + name to its metric value instead of a tensor. Must match the number of classes inferred + from the data. Default: ``None``. Examples: diff --git a/tests/ignite/metrics/test_precision.py b/tests/ignite/metrics/test_precision.py index 7bd80540f5de..3b4283ff757c 100644 --- a/tests/ignite/metrics/test_precision.py +++ b/tests/ignite/metrics/test_precision.py @@ -325,6 +325,87 @@ def test_incorrect_y_classes(average): assert pr._updated is False +@pytest.mark.parametrize( + "invalid_class_names", + [ + [0, 1, 2], # list of ints + "cat", # string instead of list + [0.1, 0.2], # list of floats + ["cat", 1], # mixed + ], +) +def test_class_names_invalid_type(invalid_class_names): + with pytest.raises(ValueError, match="class_names must be a list of strings"): + Precision(average=False, class_names=invalid_class_names) + + +@pytest.mark.parametrize("average", ["macro", "micro", "weighted", "samples", True]) +def test_class_names_incompatible_average(average): + with pytest.raises(ValueError, match="class_names is only applicable when average=False or average=None"): + Precision(average=average, class_names=["cat", "dog", "horse"]) + + +@pytest.mark.parametrize("average", [False, None]) +def test_class_names_multiclass(average): + class_names = ["cat", "dog", "horse"] + pr = Precision(average=average, class_names=class_names) + + y_pred = torch.tensor( + [ + [0.0266, 0.1719, 0.3055], + [0.6886, 0.3978, 0.8176], + [0.9230, 0.0197, 0.8395], + [0.1785, 0.2670, 0.6084], + [0.8448, 0.7177, 0.7288], + ] + ) + y = torch.tensor([2, 0, 2, 1, 0]) + + pr.update((y_pred, y)) + result = pr.compute() + + assert isinstance(result, dict) + assert list(result.keys()) == class_names + assert result == pytest.approx({"cat": 0.5, "dog": 0.0, "horse": 0.3333333333333333}) + + +def test_class_names_length_mismatch(): + pr = Precision(average=False, class_names=["cat", "dog"]) + + y_pred = torch.tensor( + [ + [0.0266, 0.1719, 0.3055], + [0.6886, 0.3978, 0.8176], + [0.9230, 0.0197, 0.8395], + ] + ) + y = torch.tensor([2, 0, 1]) + + pr.update((y_pred, y)) + with pytest.raises(ValueError, match="class_names has 2 entries but the metric computed 3 classes"): + pr.compute() + + +def test_class_names_none_returns_tensor(): + pr = Precision(average=False) + + y_pred = torch.tensor( + [ + [0.0266, 0.1719, 0.3055], + [0.6886, 0.3978, 0.8176], + [0.9230, 0.0197, 0.8395], + [0.1785, 0.2670, 0.6084], + [0.8448, 0.7177, 0.7288], + ] + ) + y = torch.tensor([2, 0, 2, 1, 0]) + + pr.update((y_pred, y)) + result = pr.compute() + + assert isinstance(result, torch.Tensor) + + @pytest.mark.usefixtures("distributed") class TestDistributed: @pytest.mark.parametrize("average", [False, "macro", "weighted", "micro"]) From 7b890cb735b3639f7028305c0699cf1a8873f9da Mon Sep 17 00:00:00 2001 From: rogueslasher Date: Thu, 16 Apr 2026 17:29:47 +0530 Subject: [PATCH 05/12] feedback: Validate class_names length in update() instead of compute() --- ignite/metrics/precision.py | 13 ++++++------- ignite/metrics/recall.py | 10 +++++++++- tests/ignite/metrics/test_precision.py | 3 +-- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/ignite/metrics/precision.py b/ignite/metrics/precision.py index 69cc84fb991b..db2669fe32b7 100644 --- a/ignite/metrics/precision.py +++ b/ignite/metrics/precision.py @@ -30,7 +30,7 @@ def __init__( raise ValueError("class_names must be a list of strings") if average is not False and average is not None: raise ValueError( - f"class_names is only applicable when average = False or None,but got average={average!r}" + f"class_names is only applicable when average=False or average=None, got average={average!r}." ) self._class_names = class_names @@ -168,11 +168,6 @@ def compute(self) -> torch.Tensor | float: return cast(torch.Tensor, fraction).mean().item() else: if self._class_names is not None: - if len(self._class_names) != fraction.shape[0]: - raise ValueError( - f"class_names has {len(self._class_names)} entries but the metric computed " - f"{fraction.shape[0]} classes." - ) return dict(zip(self._class_names, fraction.tolist())) return fraction @@ -449,5 +444,9 @@ def update(self, output: Sequence[torch.Tensor]) -> None: if self._average == "weighted": self._weight += y.sum(dim=0) - + if self._class_names is not None and len(self._class_names) != self._numerator.shape[0]: + raise ValueError( + f"class_names has {len(self._class_names)} entries but the metric computed " + f"{self._numerator.shape[0]} classes." + ) self._updated = True diff --git a/ignite/metrics/recall.py b/ignite/metrics/recall.py index 34da36ced48f..d4d12e6fc39c 100644 --- a/ignite/metrics/recall.py +++ b/ignite/metrics/recall.py @@ -97,7 +97,10 @@ class Recall(_BasePrecisionRecall): skip_unrolling: specifies whether output should be unrolled before being fed to update method. Should be true for multi-output model, for example, if ``y_pred`` contains multi-output as ``(y_pred_a, y_pred_b)`` Alternatively, ``output_transform`` can be used to handle this. - + class_names: list of class name strings used to label per-class output when ``average=False`` + or ``average=None``. If provided, :meth:`compute` returns a ``dict`` mapping each class + name to its metric value instead of a tensor. Must match the number of classes inferred + from the data. Default: ``None``. Examples: For more information on how metric works with :class:`~ignite.engine.engine.Engine`, visit :ref:`attach-engine`. @@ -240,5 +243,10 @@ def update(self, output: Sequence[torch.Tensor]) -> None: if self._average == "weighted": self._weight += y.sum(dim=0) + if self._class_names is not None and len(self._class_names) != self._numerator.shape[0]: + raise ValueError( + f"class_names has {len(self._class_names)} entries but the metric computed " + f"{self._numerator.shape[0]} classes." + ) self._updated = True diff --git a/tests/ignite/metrics/test_precision.py b/tests/ignite/metrics/test_precision.py index 3b4283ff757c..d2121858c6a3 100644 --- a/tests/ignite/metrics/test_precision.py +++ b/tests/ignite/metrics/test_precision.py @@ -381,9 +381,8 @@ def test_class_names_length_mismatch(): ) y = torch.tensor([2, 0, 1]) - pr.update((y_pred, y)) with pytest.raises(ValueError, match="class_names has 2 entries but the metric computed 3 classes"): - pr.compute() + pr.update((y_pred, y)) def test_class_names_none_returns_tensor(): From 7ea2eb50590597780ffae1ab3dc4c7ede3c80be7 Mon Sep 17 00:00:00 2001 From: rogueslasher Date: Sat, 27 Jun 2026 17:20:11 +0530 Subject: [PATCH 06/12] feat: add class_names support to Fbeta metric --- ignite/metrics/fbeta.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ignite/metrics/fbeta.py b/ignite/metrics/fbeta.py index f8c165b77903..b4cd78c8293d 100644 --- a/ignite/metrics/fbeta.py +++ b/ignite/metrics/fbeta.py @@ -177,6 +177,16 @@ def thresholded_output_transform(output): elif recall._average: raise ValueError("Input recall metric should have average=False") + if precision._class_names is not None: + + def _fbeta_with_class_names(p: dict, r: dict) -> dict: + p_vals = torch.tensor(list(p.values())) + r_vals = torch.tensor(list(r.values())) + scores = (1.0 + beta**2) * p_vals * r_vals / (beta**2 * p_vals + r_vals + 1e-15) + return dict(zip(precision._class_names, scores.tolist())) + + return MetricsLambda(_fbeta_with_class_names, precision, recall) + fbeta = (1.0 + beta**2) * precision * recall / (beta**2 * precision + recall + 1e-15) if average: From 2a3afea29d278fc77ca3b808c2e221c6734cb244 Mon Sep 17 00:00:00 2001 From: Aniket Pandey Date: Wed, 1 Jul 2026 14:51:36 +0530 Subject: [PATCH 07/12] fix: update return type annotation and docstring references for class_names --- ignite/metrics/precision.py | 6 +++--- ignite/metrics/recall.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ignite/metrics/precision.py b/ignite/metrics/precision.py index db2669fe32b7..4e6a00187ad6 100644 --- a/ignite/metrics/precision.py +++ b/ignite/metrics/precision.py @@ -135,7 +135,7 @@ def reset(self) -> None: super().reset() @sync_all_reduce("_numerator", "_denominator") - def compute(self) -> torch.Tensor | float: + def compute(self) -> torch.Tensor | float | dict: r""" Return value of the metric for `average` options `'weighted'` and `'macro'` is computed as follows. @@ -168,7 +168,7 @@ def compute(self) -> torch.Tensor | float: return cast(torch.Tensor, fraction).mean().item() else: if self._class_names is not None: - return dict(zip(self._class_names, fraction.tolist())) + return dict(zip(self._class_names, cast(torch.Tensor, fraction).tolist())) return fraction @@ -259,7 +259,7 @@ class Precision(_BasePrecisionRecall): true for multi-output model, for example, if ``y_pred`` contains multi-output as ``(y_pred_a, y_pred_b)`` Alternatively, ``output_transform`` can be used to handle this. class_names: list of class name strings used to label per-class output when ``average=False`` - or ``average=None``. If provided, :meth:`compute` returns a ``dict`` mapping each class + or ``average=None``. If provided, ``compute()`` returns a ``dict`` mapping each class name to its metric value instead of a tensor. Must match the number of classes inferred from the data. Default: ``None``. diff --git a/ignite/metrics/recall.py b/ignite/metrics/recall.py index d4d12e6fc39c..e00b9e08e84b 100644 --- a/ignite/metrics/recall.py +++ b/ignite/metrics/recall.py @@ -98,7 +98,7 @@ class Recall(_BasePrecisionRecall): true for multi-output model, for example, if ``y_pred`` contains multi-output as ``(y_pred_a, y_pred_b)`` Alternatively, ``output_transform`` can be used to handle this. class_names: list of class name strings used to label per-class output when ``average=False`` - or ``average=None``. If provided, :meth:`compute` returns a ``dict`` mapping each class + or ``average=None``. If provided, ``compute()`` returns a ``dict`` mapping each class name to its metric value instead of a tensor. Must match the number of classes inferred from the data. Default: ``None``. Examples: From 817f33b1c34321df110e023767b8beb183b69bf2 Mon Sep 17 00:00:00 2001 From: Aniket Pandey Date: Sat, 1 Aug 2026 20:43:38 +0530 Subject: [PATCH 08/12] feat(metrics): add class_names support to Precision, Recall, and Fbeta metrics --- ignite/metrics/fbeta.py | 28 +- ignite/metrics/precision.py | 14 +- ignite/metrics/recall.py | 404 ++++++++++++------------- tests/ignite/metrics/test_fbeta.py | 36 +++ tests/ignite/metrics/test_precision.py | 30 ++ tests/ignite/metrics/test_recall.py | 30 ++ 6 files changed, 332 insertions(+), 210 deletions(-) diff --git a/ignite/metrics/fbeta.py b/ignite/metrics/fbeta.py index b4cd78c8293d..4193e3499cd2 100644 --- a/ignite/metrics/fbeta.py +++ b/ignite/metrics/fbeta.py @@ -17,6 +17,7 @@ def Fbeta( recall: Recall | None = None, output_transform: Callable | None = None, device: str | torch.device | None = None, + class_names: list[str] | None = None, ) -> MetricsLambda: r"""Calculates F-beta score. @@ -42,6 +43,7 @@ def Fbeta( device: specifies which device updates are accumulated on. Setting the metric's device to be the same as your ``update`` arguments ensures the ``update`` method is non-blocking. By default, CPU. + class_names: list of class name strings used to label per-class output. Default: ``None``. Returns: MetricsLambda, F-beta metric @@ -159,31 +161,53 @@ def thresholded_output_transform(output): if precision is None and recall is None and device is None: device = torch.device("cpu") + if class_names is not None: + if not isinstance(class_names, (list, tuple)) or not all(isinstance(n, str) for n in class_names): + raise ValueError("class_names must be a list of strings") + + if precision is not None and recall is not None: + p_cn = getattr(precision, "_class_names", None) + r_cn = getattr(recall, "_class_names", None) + if p_cn is not None and r_cn is not None and p_cn != r_cn: + raise ValueError("precision and recall class_names must match") + + target_class_names = ( + class_names or getattr(precision, "_class_names", None) or getattr(recall, "_class_names", None) + ) + if precision is None: precision = Precision( output_transform=(lambda x: x) if output_transform is None else output_transform, average=False, device=cast(str | torch.device, recall._device if recall else device), + class_names=target_class_names, ) elif precision._average: raise ValueError("Input precision metric should have average=False") + elif target_class_names is not None: + precision._class_names = target_class_names if recall is None: recall = Recall( output_transform=(lambda x: x) if output_transform is None else output_transform, average=False, device=cast(str | torch.device, precision._device if precision else device), + class_names=target_class_names, ) elif recall._average: raise ValueError("Input recall metric should have average=False") + elif target_class_names is not None: + recall._class_names = target_class_names - if precision._class_names is not None: + if target_class_names is not None: def _fbeta_with_class_names(p: dict, r: dict) -> dict: p_vals = torch.tensor(list(p.values())) r_vals = torch.tensor(list(r.values())) scores = (1.0 + beta**2) * p_vals * r_vals / (beta**2 * p_vals + r_vals + 1e-15) - return dict(zip(precision._class_names, scores.tolist())) + if scores.ndim == 0: + return {target_class_names[0]: scores.item()} + return dict(zip(target_class_names, scores.tolist())) return MetricsLambda(_fbeta_with_class_names, precision, recall) diff --git a/ignite/metrics/precision.py b/ignite/metrics/precision.py index 4e6a00187ad6..9a1ad94aa705 100644 --- a/ignite/metrics/precision.py +++ b/ignite/metrics/precision.py @@ -168,6 +168,8 @@ def compute(self) -> torch.Tensor | float | dict: return cast(torch.Tensor, fraction).mean().item() else: if self._class_names is not None: + if isinstance(fraction, torch.Tensor) and fraction.ndim == 0: + return {self._class_names[0]: fraction.item()} return dict(zip(self._class_names, cast(torch.Tensor, fraction).tolist())) return fraction @@ -444,9 +446,11 @@ def update(self, output: Sequence[torch.Tensor]) -> None: if self._average == "weighted": self._weight += y.sum(dim=0) - if self._class_names is not None and len(self._class_names) != self._numerator.shape[0]: - raise ValueError( - f"class_names has {len(self._class_names)} entries but the metric computed " - f"{self._numerator.shape[0]} classes." - ) + if self._class_names is not None: + num_classes = 1 if self._numerator.ndim == 0 else self._numerator.shape[0] + if len(self._class_names) != num_classes: + raise ValueError( + f"class_names has {len(self._class_names)} entries but the metric computed " + f"{num_classes} classes." + ) self._updated = True diff --git a/ignite/metrics/recall.py b/ignite/metrics/recall.py index 95259a93c476..5b2401a71710 100644 --- a/ignite/metrics/recall.py +++ b/ignite/metrics/recall.py @@ -11,220 +11,216 @@ class Recall(_BasePrecisionRecall): r"""Calculates recall for binary, multiclass and multilabel data. - .. math:: \text{Recall} = \frac{ TP }{ TP + FN } + .. math:: \text{Recall} = \frac{ TP }{ TP + FN } - where :math:`\text{TP}` is true positives and :math:`\text{FN}` is false negatives. + where :math:`\text{TP}` is true positives and :math:`\text{FN}` is false negatives. - - ``update`` must receive output of the form ``(y_pred, y)``. - - `y_pred` must be in the following shape (batch_size, num_categories, ...) or (batch_size, ...). - - `y` must be in the following shape (batch_size, ...). + - ``update`` must receive output of the form ``(y_pred, y)``. + - `y_pred` must be in the following shape (batch_size, num_categories, ...) or (batch_size, ...). + - `y` must be in the following shape (batch_size, ...). - Args: - output_transform: a callable that is used to transform the - :class:`~ignite.engine.engine.Engine`'s ``process_function``'s output into the - form expected by the metric. This can be useful if, for example, you have a multi-output model and - you want to compute the metric with respect to one of the outputs. - average: available options are - - False - default option. For multiclass and multilabel inputs, per class and per label - metric is returned respectively. + Args: + output_transform: a callable that is used to transform the + :class:`~ignite.engine.engine.Engine`'s ``process_function``'s output into the + form expected by the metric. This can be useful if, for example, you have a multi-output model and + you want to compute the metric with respect to one of the outputs. + average: available options are - None - like `False` option except that per class metric is returned for binary data as well. - For compatibility with Scikit-Learn api. + False + default option. For multiclass and multilabel inputs, per class and per label + metric is returned respectively. - 'micro' - Metric is computed counting stats of classes/labels altogether. + None + like `False` option except that per class metric is returned for binary data as well. + For compatibility with Scikit-Learn api. - .. math:: - \text{Micro Recall} = \frac{\sum_{k=1}^C TP_k}{\sum_{k=1}^C TP_k+FN_k} + 'micro' + Metric is computed counting stats of classes/labels altogether. - where :math:`C` is the number of classes/labels (2 in binary case). :math:`k` in - :math:`TP_k` and :math:`FN_k` means that the measures are computed for class/label :math:`k` (in - a one-vs-rest sense in multiclass case). + .. math:: + \text{Micro Recall} = \frac{\sum_{k=1}^C TP_k}{\sum_{k=1}^C TP_k+FN_k} - For binary and multiclass inputs, this is equivalent with accuracy, - so use :class:`~ignite.metrics.accuracy.Accuracy`. + where :math:`C` is the number of classes/labels (2 in binary case). :math:`k` in + :math:`TP_k` and :math:`FN_k` means that the measures are computed for class/label :math:`k` (in + a one-vs-rest sense in multiclass case). - 'samples' - for multilabel input, at first, recall is computed on a - per sample basis and then average across samples is returned. - - .. math:: - \text{Sample-averaged Recall} = \frac{\sum_{n=1}^N \frac{TP_n}{TP_n+FN_n}}{N} + For binary and multiclass inputs, this is equivalent with accuracy, + so use :class:`~ignite.metrics.accuracy.Accuracy`. - where :math:`N` is the number of samples. :math:`n` in :math:`TP_n` and :math:`FN_n` - means that the measures are computed for sample :math:`n`, across labels. + 'samples' + for multilabel input, at first, recall is computed on a + per sample basis and then average across samples is returned. - Incompatible with binary and multiclass inputs. + .. math:: + \text{Sample-averaged Recall} = \frac{\sum_{n=1}^N \frac{TP_n}{TP_n+FN_n}}{N} - 'weighted' - like macro recall but considers class/label imbalance. For binary and multiclass - input, it computes metric for each class then returns average of them weighted by - support of classes (number of actual samples in each class). For multilabel input, - it computes recall for each label then returns average of them weighted by support - of labels (number of actual positive samples in each label). + where :math:`N` is the number of samples. :math:`n` in :math:`TP_n` and :math:`FN_n` + means that the measures are computed for sample :math:`n`, across labels. - .. math:: - Recall_k = \frac{TP_k}{TP_k+FN_k} + Incompatible with binary and multiclass inputs. - .. math:: - \text{Weighted Recall} = \frac{\sum_{k=1}^C P_k * Recall_k}{N} + 'weighted' + like macro recall but considers class/label imbalance. For binary and multiclass + input, it computes metric for each class then returns average of them weighted by + support of classes (number of actual samples in each class). For multilabel input, + it computes recall for each label then returns average of them weighted by support + of labels (number of actual positive samples in each label). - where :math:`C` is the number of classes (2 in binary case). :math:`P_k` is the number - of samples belonged to class :math:`k` in binary and multiclass case, and the number of - positive samples belonged to label :math:`k` in multilabel case. + .. math:: + Recall_k = \frac{TP_k}{TP_k+FN_k} - Note that for binary and multiclass data, weighted recall is equivalent - with accuracy, so use :class:`~ignite.metrics.accuracy.Accuracy`. - - macro - computes macro recall which is unweighted average of metric computed across - classes or labels. - - .. math:: - \text{Macro Recall} = \frac{\sum_{k=1}^C Recall_k}{C} - - where :math:`C` is the number of classes (2 in binary case). - - True - like macro option. For backward compatibility. - is_multilabel: flag to use in multilabel case. By default, value is False. - device: specifies which device updates are accumulated on. Setting the metric's - device to be the same as your ``update`` arguments ensures the ``update`` method is non-blocking. By - default, CPU. - skip_unrolling: specifies whether output should be unrolled before being fed to update method. Should be - true for multi-output model, for example, if ``y_pred`` contains multi-output as ``(y_pred_a, y_pred_b)`` - Alternatively, ``output_transform`` can be used to handle this. - class_names: list of class name strings used to label per-class output when ``average=False`` - <<<<<<< HEAD - or ``average=None``. If provided, ``compute()`` returns a ``dict`` mapping each class - ======= - or ``average=None``. If provided, :meth:`compute` returns a ``dict`` mapping each class - >>>>>>> 7ea2eb50590597780ffae1ab3dc4c7ede3c80be7 - name to its metric value instead of a tensor. Must match the number of classes inferred - from the data. Default: ``None``. - Examples: - - For more information on how metric works with :class:`~ignite.engine.engine.Engine`, visit :ref:`attach-engine`. - - .. include:: defaults.rst - :start-after: :orphan: - - Binary case. In binary and multilabel cases, the elements of - `y` and `y_pred` should have 0 or 1 values. - - .. testcode:: 1 - - metric = Recall() - two_class_metric = Recall(average=None) # Returns recall for both classes - metric.attach(default_evaluator, "recall") - two_class_metric.attach(default_evaluator, "both classes recall") - y_true = torch.tensor([1, 0, 1, 1, 0, 1]) - y_pred = torch.tensor([1, 0, 1, 0, 1, 1]) - state = default_evaluator.run([[y_pred, y_true]]) - print(f"Recall: {state.metrics['recall']}") - print(f"Recall for class 0 and class 1: {state.metrics['both classes recall']}") - - .. testoutput:: 1 - - Recall: 0.75 - Recall for class 0 and class 1: tensor([0.5000, 0.7500], dtype=torch.float64) - - Multiclass case - - .. testcode:: 2 - - metric = Recall() - macro_metric = Recall(average=True) - - metric.attach(default_evaluator, "recall") - macro_metric.attach(default_evaluator, "macro recall") - - y_true = torch.tensor([2, 0, 2, 1, 0]) - y_pred = torch.tensor([ - [0.0266, 0.1719, 0.3055], - [0.6886, 0.3978, 0.8176], - [0.9230, 0.0197, 0.8395], - [0.1785, 0.2670, 0.6084], - [0.8448, 0.7177, 0.7288] - ]) - state = default_evaluator.run([[y_pred, y_true]]) - print(f"Recall: {state.metrics['recall']}") - print(f"Macro Recall: {state.metrics['macro recall']}") - - .. testoutput:: 2 - - Recall: tensor([0.5000, 0.0000, 0.5000], dtype=torch.float64) - Macro Recall: 0.3333333333333333 - - Multilabel case, the shapes must be (batch_size, num_categories, ...) - - .. testcode:: 3 - - metric = Recall(is_multilabel=True) - micro_metric = Recall(is_multilabel=True, average='micro') - macro_metric = Recall(is_multilabel=True, average=True) - samples_metric = Recall(is_multilabel=True, average='samples') - - metric.attach(default_evaluator, "recall") - micro_metric.attach(default_evaluator, "micro recall") - macro_metric.attach(default_evaluator, "macro recall") - samples_metric.attach(default_evaluator, "samples recall") - - y_true = torch.tensor([ - [0, 0, 1], - [0, 0, 0], - [0, 0, 0], - [1, 0, 0], - [0, 1, 1], - ]) - y_pred = torch.tensor([ - [1, 1, 0], - [1, 0, 1], - [1, 0, 0], - [1, 0, 1], - [1, 1, 0], - ]) - state = default_evaluator.run([[y_pred, y_true]]) - print(f"Recall: {state.metrics['recall']}") - print(f"Micro Recall: {state.metrics['micro recall']}") - print(f"Macro Recall: {state.metrics['macro recall']}") - print(f"Samples Recall: {state.metrics['samples recall']}") - - .. testoutput:: 3 - - Recall: tensor([1., 1., 0.], dtype=torch.float64) - Micro Recall: 0.5 - Macro Recall: 0.6666666666666666 - Samples Recall: 0.3 - - Thresholding of predictions can be done as below: - - .. testcode:: 4 - - def thresholded_output_transform(output): - y_pred, y = output - y_pred = torch.round(y_pred) - return y_pred, y - - metric = Recall(output_transform=thresholded_output_transform) - metric.attach(default_evaluator, "recall") - y_true = torch.tensor([1, 0, 1, 1, 0, 1]) - y_pred = torch.tensor([0.6, 0.2, 0.9, 0.4, 0.7, 0.65]) - state = default_evaluator.run([[y_pred, y_true]]) - print(state.metrics['recall']) - - .. testoutput:: 4 - - 0.75 - - .. versionchanged:: 0.4.10 - Some new options were added to `average` parameter. - - .. versionchanged:: 0.5.1 - ``skip_unrolling`` argument is added. + .. math:: + \text{Weighted Recall} = \frac{\sum_{k=1}^C P_k * Recall_k}{N} + + where :math:`C` is the number of classes (2 in binary case). :math:`P_k` is the number + of samples belonged to class :math:`k` in binary and multiclass case, and the number of + positive samples belonged to label :math:`k` in multilabel case. + + Note that for binary and multiclass data, weighted recall is equivalent + with accuracy, so use :class:`~ignite.metrics.accuracy.Accuracy`. + + macro + computes macro recall which is unweighted average of metric computed across + classes or labels. + + .. math:: + \text{Macro Recall} = \frac{\sum_{k=1}^C Recall_k}{C} + + where :math:`C` is the number of classes (2 in binary case). + + True + like macro option. For backward compatibility. + is_multilabel: flag to use in multilabel case. By default, value is False. + device: specifies which device updates are accumulated on. Setting the metric's + device to be the same as your ``update`` arguments ensures the ``update`` method is non-blocking. By + default, CPU. + skip_unrolling: specifies whether output should be unrolled before being fed to update method. Should be + true for multi-output model, for example, if ``y_pred`` contains multi-output as ``(y_pred_a, y_pred_b)`` + Alternatively, ``output_transform`` can be used to handle this. + class_names: list of class name strings used to label per-class output when ``average=False`` + or ``average=None``. If provided, ``compute()`` returns a ``dict`` mapping each class + name to its metric value instead of a tensor. Must match the number of classes inferred + from the data. Default: ``None``. + Examples: + + For more information on how metric works with :class:`~ignite.engine.engine.Engine`, visit :ref:`attach-engine`. + + .. include:: defaults.rst + :start-after: :orphan: + + Binary case. In binary and multilabel cases, the elements of + `y` and `y_pred` should have 0 or 1 values. + + .. testcode:: 1 + + metric = Recall() + two_class_metric = Recall(average=None) # Returns recall for both classes + metric.attach(default_evaluator, "recall") + two_class_metric.attach(default_evaluator, "both classes recall") + y_true = torch.tensor([1, 0, 1, 1, 0, 1]) + y_pred = torch.tensor([1, 0, 1, 0, 1, 1]) + state = default_evaluator.run([[y_pred, y_true]]) + print(f"Recall: {state.metrics['recall']}") + print(f"Recall for class 0 and class 1: {state.metrics['both classes recall']}") + + .. testoutput:: 1 + + Recall: 0.75 + Recall for class 0 and class 1: tensor([0.5000, 0.7500], dtype=torch.float64) + + Multiclass case + + .. testcode:: 2 + + metric = Recall() + macro_metric = Recall(average=True) + + metric.attach(default_evaluator, "recall") + macro_metric.attach(default_evaluator, "macro recall") + + y_true = torch.tensor([2, 0, 2, 1, 0]) + y_pred = torch.tensor([ + [0.0266, 0.1719, 0.3055], + [0.6886, 0.3978, 0.8176], + [0.9230, 0.0197, 0.8395], + [0.1785, 0.2670, 0.6084], + [0.8448, 0.7177, 0.7288] + ]) + state = default_evaluator.run([[y_pred, y_true]]) + print(f"Recall: {state.metrics['recall']}") + print(f"Macro Recall: {state.metrics['macro recall']}") + + .. testoutput:: 2 + + Recall: tensor([0.5000, 0.0000, 0.5000], dtype=torch.float64) + Macro Recall: 0.3333333333333333 + + Multilabel case, the shapes must be (batch_size, num_categories, ...) + + .. testcode:: 3 + + metric = Recall(is_multilabel=True) + micro_metric = Recall(is_multilabel=True, average='micro') + macro_metric = Recall(is_multilabel=True, average=True) + samples_metric = Recall(is_multilabel=True, average='samples') + + metric.attach(default_evaluator, "recall") + micro_metric.attach(default_evaluator, "micro recall") + macro_metric.attach(default_evaluator, "macro recall") + samples_metric.attach(default_evaluator, "samples recall") + + y_true = torch.tensor([ + [0, 0, 1], + [0, 0, 0], + [0, 0, 0], + [1, 0, 0], + [0, 1, 1], + ]) + y_pred = torch.tensor([ + [1, 1, 0], + [1, 0, 1], + [1, 0, 0], + [1, 0, 1], + [1, 1, 0], + ]) + state = default_evaluator.run([[y_pred, y_true]]) + print(f"Recall: {state.metrics['recall']}") + print(f"Micro Recall: {state.metrics['micro recall']}") + print(f"Macro Recall: {state.metrics['macro recall']}") + print(f"Samples Recall: {state.metrics['samples recall']}") + + .. testoutput:: 3 + + Recall: tensor([1., 1., 0.], dtype=torch.float64) + Micro Recall: 0.5 + Macro Recall: 0.6666666666666666 + Samples Recall: 0.3 + + Thresholding of predictions can be done as below: + + .. testcode:: 4 + + def thresholded_output_transform(output): + y_pred, y = output + y_pred = torch.round(y_pred) + return y_pred, y + + metric = Recall(output_transform=thresholded_output_transform) + metric.attach(default_evaluator, "recall") + y_true = torch.tensor([1, 0, 1, 1, 0, 1]) + y_pred = torch.tensor([0.6, 0.2, 0.9, 0.4, 0.7, 0.65]) + state = default_evaluator.run([[y_pred, y_true]]) + print(state.metrics['recall']) + + .. testoutput:: 4 + + 0.75 + + .. versionchanged:: 0.4.10 + Some new options were added to `average` parameter. + + .. versionchanged:: 0.5.1 + ``skip_unrolling`` argument is added. """ @reinit__is_reduced @@ -247,10 +243,12 @@ def update(self, output: Sequence[torch.Tensor]) -> None: if self._average == "weighted": self._weight += y.sum(dim=0) - if self._class_names is not None and len(self._class_names) != self._numerator.shape[0]: - raise ValueError( - f"class_names has {len(self._class_names)} entries but the metric computed " - f"{self._numerator.shape[0]} classes." - ) + if self._class_names is not None: + num_classes = 1 if self._numerator.ndim == 0 else self._numerator.shape[0] + if len(self._class_names) != num_classes: + raise ValueError( + f"class_names has {len(self._class_names)} entries but the metric computed " + f"{num_classes} classes." + ) self._updated = True diff --git a/tests/ignite/metrics/test_fbeta.py b/tests/ignite/metrics/test_fbeta.py index 251ff590cfd4..601c021b1939 100644 --- a/tests/ignite/metrics/test_fbeta.py +++ b/tests/ignite/metrics/test_fbeta.py @@ -233,3 +233,39 @@ def test_multinode_distrib_gloo_cpu_or_gpu(distributed_context_multi_node_gloo): def test_multinode_distrib_nccl_gpu(distributed_context_multi_node_nccl): device = idist.device() _test_distrib_integration(device) + + +def test_class_names(): + # Invalid class_names type + with pytest.raises(ValueError, match="class_names must be a list of strings"): + Fbeta(beta=1.0, class_names=[1, 2]) + + # Mismatched precision and recall class_names + p = Precision(average=False, class_names=["cat", "dog"]) + r = Recall(average=False, class_names=["a", "b"]) + with pytest.raises(ValueError, match="precision and recall class_names must match"): + Fbeta(beta=1.0, precision=p, recall=r) + + # Correct computation passing class_names directly to Fbeta + f1 = Fbeta(beta=1.0, class_names=["cat", "dog", "bird"]) + y_true = torch.tensor([0, 1, 2]) + y_pred = torch.tensor( + [ + [1.0, 0.0, 0.0], + [0.0, 1.0, 0.0], + [0.0, 0.0, 1.0], + ] + ) + f1.update((y_pred, y_true)) + res = f1.compute() + assert isinstance(res, dict) + assert res == {"cat": 1.0, "dog": 1.0, "bird": 1.0} + + # Correct computation with Precision and Recall having class_names + p2 = Precision(average=False, class_names=["cat", "dog", "bird"]) + r2 = Recall(average=False, class_names=["cat", "dog", "bird"]) + f2 = Fbeta(beta=1.0, precision=p2, recall=r2) + f2.update((y_pred, y_true)) + res2 = f2.compute() + assert isinstance(res2, dict) + assert res2 == {"cat": 1.0, "dog": 1.0, "bird": 1.0} diff --git a/tests/ignite/metrics/test_precision.py b/tests/ignite/metrics/test_precision.py index d2121858c6a3..441670c46353 100644 --- a/tests/ignite/metrics/test_precision.py +++ b/tests/ignite/metrics/test_precision.py @@ -583,3 +583,33 @@ def test_multilabel_accumulator_device(self, average): if average == "weighted": assert pr._weight.device == metric_device, f"{type(pr._weight.device)}:{pr._weight.device} vs " f"{type(metric_device)}:{metric_device}" + + +def test_class_names(): + # Invalid class_names type + with pytest.raises(ValueError, match="class_names must be a list of strings"): + Precision(average=False, class_names=[1, 2]) + + # Incompatible average mode + with pytest.raises(ValueError, match="class_names is only applicable when average=False or average=None"): + Precision(average="macro", class_names=["cat", "dog"]) + + # Correct computation returning dict + pr = Precision(average=False, class_names=["cat", "dog", "bird"]) + y_true = torch.tensor([0, 1, 2]) + y_pred = torch.tensor( + [ + [1.0, 0.0, 0.0], + [0.0, 1.0, 0.0], + [0.0, 0.0, 1.0], + ] + ) + pr.update((y_pred, y_true)) + res = pr.compute() + assert isinstance(res, dict) + assert res == {"cat": 1.0, "dog": 1.0, "bird": 1.0} + + # Class names length mismatch + pr_mismatch = Precision(average=False, class_names=["cat", "dog"]) + with pytest.raises(ValueError, match="class_names has 2 entries but the metric computed 3 classes."): + pr_mismatch.update((y_pred, y_true)) diff --git a/tests/ignite/metrics/test_recall.py b/tests/ignite/metrics/test_recall.py index 90e78d1a82ad..df29278a7c79 100644 --- a/tests/ignite/metrics/test_recall.py +++ b/tests/ignite/metrics/test_recall.py @@ -505,3 +505,33 @@ def test_multilabel_accumulator_device(self, average): if average == "weighted": assert re._weight.device == metric_device, f"{type(re._weight.device)}:{re._weight.device} vs " f"{type(metric_device)}:{metric_device}" + + +def test_class_names(): + # Invalid class_names type + with pytest.raises(ValueError, match="class_names must be a list of strings"): + Recall(average=False, class_names=[1, 2]) + + # Incompatible average mode + with pytest.raises(ValueError, match="class_names is only applicable when average=False or average=None"): + Recall(average="macro", class_names=["cat", "dog"]) + + # Correct computation returning dict + re = Recall(average=False, class_names=["cat", "dog", "bird"]) + y_true = torch.tensor([0, 1, 2]) + y_pred = torch.tensor( + [ + [1.0, 0.0, 0.0], + [0.0, 1.0, 0.0], + [0.0, 0.0, 1.0], + ] + ) + re.update((y_pred, y_true)) + res = re.compute() + assert isinstance(res, dict) + assert res == {"cat": 1.0, "dog": 1.0, "bird": 1.0} + + # Class names length mismatch + re_mismatch = Recall(average=False, class_names=["cat", "dog"]) + with pytest.raises(ValueError, match="class_names has 2 entries but the metric computed 3 classes."): + re_mismatch.update((y_pred, y_true)) From 7e12121626179dbe9389fd93218cee235c2b16e1 Mon Sep 17 00:00:00 2001 From: Aniket Pandey Date: Mon, 3 Aug 2026 16:17:03 +0530 Subject: [PATCH 09/12] fix(metrics): enforce explicit class_names matching and early average check in Fbeta --- ignite/metrics/fbeta.py | 33 +++++++++++++++++------------- tests/ignite/metrics/test_fbeta.py | 22 ++++++++++++++++---- 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/ignite/metrics/fbeta.py b/ignite/metrics/fbeta.py index 4193e3499cd2..a06874e73f3c 100644 --- a/ignite/metrics/fbeta.py +++ b/ignite/metrics/fbeta.py @@ -165,16 +165,29 @@ def thresholded_output_transform(output): if not isinstance(class_names, (list, tuple)) or not all(isinstance(n, str) for n in class_names): raise ValueError("class_names must be a list of strings") - if precision is not None and recall is not None: - p_cn = getattr(precision, "_class_names", None) - r_cn = getattr(recall, "_class_names", None) - if p_cn is not None and r_cn is not None and p_cn != r_cn: - raise ValueError("precision and recall class_names must match") - target_class_names = ( class_names or getattr(precision, "_class_names", None) or getattr(recall, "_class_names", None) ) + if target_class_names is not None and average is not False and average is not None: + raise ValueError(f"class_names is only applicable when average=False or average=None, got average={average!r}.") + + if precision is not None: + if precision._average: + raise ValueError("Input precision metric should have average=False") + if class_names is not None and precision._class_names != class_names: + raise ValueError("precision metric class_names must match Fbeta class_names") + + if recall is not None: + if recall._average: + raise ValueError("Input recall metric should have average=False") + if class_names is not None and recall._class_names != class_names: + raise ValueError("recall metric class_names must match Fbeta class_names") + + if precision is not None and recall is not None: + if precision._class_names != recall._class_names: + raise ValueError("precision and recall class_names must match") + if precision is None: precision = Precision( output_transform=(lambda x: x) if output_transform is None else output_transform, @@ -182,10 +195,6 @@ def thresholded_output_transform(output): device=cast(str | torch.device, recall._device if recall else device), class_names=target_class_names, ) - elif precision._average: - raise ValueError("Input precision metric should have average=False") - elif target_class_names is not None: - precision._class_names = target_class_names if recall is None: recall = Recall( @@ -194,10 +203,6 @@ def thresholded_output_transform(output): device=cast(str | torch.device, precision._device if precision else device), class_names=target_class_names, ) - elif recall._average: - raise ValueError("Input recall metric should have average=False") - elif target_class_names is not None: - recall._class_names = target_class_names if target_class_names is not None: diff --git a/tests/ignite/metrics/test_fbeta.py b/tests/ignite/metrics/test_fbeta.py index 601c021b1939..b979dd909def 100644 --- a/tests/ignite/metrics/test_fbeta.py +++ b/tests/ignite/metrics/test_fbeta.py @@ -238,16 +238,30 @@ def test_multinode_distrib_nccl_gpu(distributed_context_multi_node_nccl): def test_class_names(): # Invalid class_names type with pytest.raises(ValueError, match="class_names must be a list of strings"): - Fbeta(beta=1.0, class_names=[1, 2]) + Fbeta(beta=1.0, average=False, class_names=[1, 2]) + + # Early check for average=True when class_names is given + with pytest.raises(ValueError, match="class_names is only applicable when average=False or average=None"): + Fbeta(beta=1.0, average=True, class_names=["cat", "dog"]) + + # Precision metric without class_names passed to Fbeta with class_names + p_no_cn = Precision(average=False) + with pytest.raises(ValueError, match="precision metric class_names must match Fbeta class_names"): + Fbeta(beta=1.0, average=False, class_names=["cat", "dog"], precision=p_no_cn) + + # Recall metric without class_names passed to Fbeta with class_names + r_no_cn = Recall(average=False) + with pytest.raises(ValueError, match="recall metric class_names must match Fbeta class_names"): + Fbeta(beta=1.0, average=False, class_names=["cat", "dog"], recall=r_no_cn) # Mismatched precision and recall class_names p = Precision(average=False, class_names=["cat", "dog"]) r = Recall(average=False, class_names=["a", "b"]) with pytest.raises(ValueError, match="precision and recall class_names must match"): - Fbeta(beta=1.0, precision=p, recall=r) + Fbeta(beta=1.0, average=False, precision=p, recall=r) # Correct computation passing class_names directly to Fbeta - f1 = Fbeta(beta=1.0, class_names=["cat", "dog", "bird"]) + f1 = Fbeta(beta=1.0, average=False, class_names=["cat", "dog", "bird"]) y_true = torch.tensor([0, 1, 2]) y_pred = torch.tensor( [ @@ -264,7 +278,7 @@ def test_class_names(): # Correct computation with Precision and Recall having class_names p2 = Precision(average=False, class_names=["cat", "dog", "bird"]) r2 = Recall(average=False, class_names=["cat", "dog", "bird"]) - f2 = Fbeta(beta=1.0, precision=p2, recall=r2) + f2 = Fbeta(beta=1.0, average=False, precision=p2, recall=r2) f2.update((y_pred, y_true)) res2 = f2.compute() assert isinstance(res2, dict) From 7b8ce312670c31fcd036f1d8e91eb97c79d0fcba Mon Sep 17 00:00:00 2001 From: Aniket Pandey Date: Thu, 6 Aug 2026 13:33:26 +0530 Subject: [PATCH 10/12] refactor(metrics): simplify Fbeta class_names validation with active_metrics pattern --- ignite/metrics/fbeta.py | 39 +++++++++++++++--------------- tests/ignite/metrics/test_fbeta.py | 13 +++++++--- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/ignite/metrics/fbeta.py b/ignite/metrics/fbeta.py index a06874e73f3c..56817cb37f40 100644 --- a/ignite/metrics/fbeta.py +++ b/ignite/metrics/fbeta.py @@ -164,29 +164,30 @@ def thresholded_output_transform(output): if class_names is not None: if not isinstance(class_names, (list, tuple)) or not all(isinstance(n, str) for n in class_names): raise ValueError("class_names must be a list of strings") + if average is not False and average is not None: + raise ValueError( + f"class_names is only applicable when average=False or average=None, got average={average!r}." + ) - target_class_names = ( - class_names or getattr(precision, "_class_names", None) or getattr(recall, "_class_names", None) - ) + active_metrics = [m for m in (precision, recall) if m is not None] - if target_class_names is not None and average is not False and average is not None: - raise ValueError(f"class_names is only applicable when average=False or average=None, got average={average!r}.") + if any(m._average for m in active_metrics): + raise ValueError("Input precision and recall metrics should have average=False") - if precision is not None: - if precision._average: - raise ValueError("Input precision metric should have average=False") - if class_names is not None and precision._class_names != class_names: - raise ValueError("precision metric class_names must match Fbeta class_names") + if class_names is not None and any(m._class_names != class_names for m in active_metrics): + raise ValueError("precision and recall metric class_names must match Fbeta class_names") - if recall is not None: - if recall._average: - raise ValueError("Input recall metric should have average=False") - if class_names is not None and recall._class_names != class_names: - raise ValueError("recall metric class_names must match Fbeta class_names") - - if precision is not None and recall is not None: - if precision._class_names != recall._class_names: - raise ValueError("precision and recall class_names must match") + if len(active_metrics) == 2 and active_metrics[0]._class_names != active_metrics[1]._class_names: + raise ValueError("precision and recall class_names must match") + + target_class_names = class_names + if target_class_names is None and precision is not None: + target_class_names = precision._class_names + if target_class_names is None and recall is not None: + target_class_names = recall._class_names + + if target_class_names is not None and average is not False and average is not None: + raise ValueError(f"class_names is only applicable when average=False or average=None, got average={average!r}.") if precision is None: precision = Precision( diff --git a/tests/ignite/metrics/test_fbeta.py b/tests/ignite/metrics/test_fbeta.py index b979dd909def..26a260f70dc7 100644 --- a/tests/ignite/metrics/test_fbeta.py +++ b/tests/ignite/metrics/test_fbeta.py @@ -15,11 +15,11 @@ def test_wrong_inputs(): with pytest.raises(ValueError, match=r"Beta should be a positive integer"): Fbeta(0.0) - with pytest.raises(ValueError, match=r"Input precision metric should have average=False"): + with pytest.raises(ValueError, match=r"Input precision and recall metrics should have average=False"): p = Precision(average="micro") Fbeta(1.0, precision=p) - with pytest.raises(ValueError, match=r"Input recall metric should have average=False"): + with pytest.raises(ValueError, match=r"Input precision and recall metrics should have average=False"): r = Recall(average="samples") Fbeta(1.0, recall=r) @@ -246,12 +246,12 @@ def test_class_names(): # Precision metric without class_names passed to Fbeta with class_names p_no_cn = Precision(average=False) - with pytest.raises(ValueError, match="precision metric class_names must match Fbeta class_names"): + with pytest.raises(ValueError, match="precision and recall metric class_names must match Fbeta class_names"): Fbeta(beta=1.0, average=False, class_names=["cat", "dog"], precision=p_no_cn) # Recall metric without class_names passed to Fbeta with class_names r_no_cn = Recall(average=False) - with pytest.raises(ValueError, match="recall metric class_names must match Fbeta class_names"): + with pytest.raises(ValueError, match="precision and recall metric class_names must match Fbeta class_names"): Fbeta(beta=1.0, average=False, class_names=["cat", "dog"], recall=r_no_cn) # Mismatched precision and recall class_names @@ -260,6 +260,11 @@ def test_class_names(): with pytest.raises(ValueError, match="precision and recall class_names must match"): Fbeta(beta=1.0, average=False, precision=p, recall=r) + # Input precision metric with average != False + p_avg = Precision(average="macro") + with pytest.raises(ValueError, match="Input precision and recall metrics should have average=False"): + Fbeta(beta=1.0, average=False, precision=p_avg) + # Correct computation passing class_names directly to Fbeta f1 = Fbeta(beta=1.0, average=False, class_names=["cat", "dog", "bird"]) y_true = torch.tensor([0, 1, 2]) From 1e3bdb84f67f09478e9be6b187b3ab3032470c7a Mon Sep 17 00:00:00 2001 From: Aniket Pandey Date: Sun, 9 Aug 2026 18:26:33 +0530 Subject: [PATCH 11/12] refactor(metrics): remove redundant average check in Fbeta --- ignite/metrics/fbeta.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ignite/metrics/fbeta.py b/ignite/metrics/fbeta.py index 56817cb37f40..728d2df738e8 100644 --- a/ignite/metrics/fbeta.py +++ b/ignite/metrics/fbeta.py @@ -186,9 +186,6 @@ def thresholded_output_transform(output): if target_class_names is None and recall is not None: target_class_names = recall._class_names - if target_class_names is not None and average is not False and average is not None: - raise ValueError(f"class_names is only applicable when average=False or average=None, got average={average!r}.") - if precision is None: precision = Precision( output_transform=(lambda x: x) if output_transform is None else output_transform, From 2f8887ce9577af9ec4f1b2c47226cdf431e2bfb8 Mon Sep 17 00:00:00 2001 From: Aniket Pandey Date: Tue, 11 Aug 2026 13:35:21 +0530 Subject: [PATCH 12/12] docs(metrics): add versionadded 0.6.0 tags for class_names parameter --- ignite/metrics/fbeta.py | 2 ++ ignite/metrics/precision.py | 5 +++++ ignite/metrics/recall.py | 6 ++++++ 3 files changed, 13 insertions(+) diff --git a/ignite/metrics/fbeta.py b/ignite/metrics/fbeta.py index 728d2df738e8..bb975a152914 100644 --- a/ignite/metrics/fbeta.py +++ b/ignite/metrics/fbeta.py @@ -45,6 +45,8 @@ def Fbeta( default, CPU. class_names: list of class name strings used to label per-class output. Default: ``None``. + .. versionadded:: 0.6.0 + Returns: MetricsLambda, F-beta metric diff --git a/ignite/metrics/precision.py b/ignite/metrics/precision.py index 9a1ad94aa705..65ef320d7a5a 100644 --- a/ignite/metrics/precision.py +++ b/ignite/metrics/precision.py @@ -265,6 +265,8 @@ class Precision(_BasePrecisionRecall): name to its metric value instead of a tensor. Must match the number of classes inferred from the data. Default: ``None``. + .. versionadded:: 0.6.0 + Examples: For more information on how metric works with :class:`~ignite.engine.engine.Engine`, visit :ref:`attach-engine`. @@ -397,6 +399,9 @@ def thresholded_output_transform(output): .. versionchanged:: 0.5.1 ``skip_unrolling`` argument is added. + + .. versionchanged:: 0.6.0 + ``class_names`` argument is added. """ @reinit__is_reduced diff --git a/ignite/metrics/recall.py b/ignite/metrics/recall.py index 5b2401a71710..0f68b6537930 100644 --- a/ignite/metrics/recall.py +++ b/ignite/metrics/recall.py @@ -101,6 +101,9 @@ class Recall(_BasePrecisionRecall): or ``average=None``. If provided, ``compute()`` returns a ``dict`` mapping each class name to its metric value instead of a tensor. Must match the number of classes inferred from the data. Default: ``None``. + + .. versionadded:: 0.6.0 + Examples: For more information on how metric works with :class:`~ignite.engine.engine.Engine`, visit :ref:`attach-engine`. @@ -221,6 +224,9 @@ def thresholded_output_transform(output): .. versionchanged:: 0.5.1 ``skip_unrolling`` argument is added. + + .. versionchanged:: 0.6.0 + ``class_names`` argument is added. """ @reinit__is_reduced