diff --git a/packages/jabs-core/src/jabs/core/constants.py b/packages/jabs-core/src/jabs/core/constants.py index 0b6a9bd1..92d86eac 100644 --- a/packages/jabs-core/src/jabs/core/constants.py +++ b/packages/jabs-core/src/jabs/core/constants.py @@ -18,5 +18,11 @@ CLASSIFIER_MODE_KEY = "classifier_mode" CACHE_FORMAT_KEY = "cache_format" +# behavior-scoped settings keys stored under the "behavior" section of project.json +# ordered list of prediction postprocessing stage configurations +POSTPROCESSING_KEY = "postprocessing" +# when true, cross-validation also reports metrics with the postprocessing pipeline applied +EVALUATE_POSTPROCESSING_IN_CV_KEY = "evaluate_postprocessing_in_cv" + # reserved behavior name used in multi-class mode to store explicit negative labels MULTICLASS_NONE_BEHAVIOR = "None" diff --git a/src/jabs/classifier/__init__.py b/src/jabs/classifier/__init__.py index 993caa9f..d311ad11 100644 --- a/src/jabs/classifier/__init__.py +++ b/src/jabs/classifier/__init__.py @@ -7,6 +7,12 @@ from .classifier import Classifier from .cross_validation import run_leave_one_group_out_cv +from .cv_postprocessing import ( + FoldPostprocessingEvaluation, + enabled_stage_configs, + evaluate_group_with_postprocessing, +) +from .inference import IdentityPrediction, predict_identity from .mlflow_logging import ( MlflowLoggingError, log_cross_validation_to_mlflow, @@ -19,6 +25,7 @@ BinaryCVResult, CrossValidationResult, MultiClassCVResult, + PostprocessedMetrics, TrainingReportData, generate_markdown_report, save_training_report, @@ -29,14 +36,20 @@ "Classifier", "ClassifierProtocol", "CrossValidationResult", + "FoldPostprocessingEvaluation", + "IdentityPrediction", "MlflowLoggingError", "MultiClassCVResult", "MultiClassClassifier", + "PostprocessedMetrics", "TrainingReportData", + "enabled_stage_configs", + "evaluate_group_with_postprocessing", "generate_markdown_report", "log_cross_validation_to_mlflow", "mlflow_available", "parse_kv_tags", + "predict_identity", "run_leave_one_group_out_cv", "save_training_report", ] diff --git a/src/jabs/classifier/cross_validation.py b/src/jabs/classifier/cross_validation.py index 6ea2caeb..93b9cf65 100644 --- a/src/jabs/classifier/cross_validation.py +++ b/src/jabs/classifier/cross_validation.py @@ -1,22 +1,41 @@ """Cross-validation utilities for JABS classifier training.""" +import logging from collections.abc import Callable from typing import TYPE_CHECKING, NotRequired, TypedDict import numpy as np import numpy.typing as npt import pandas as pd +from sklearn.metrics import confusion_matrix as sk_confusion_matrix from sklearn.metrics import precision_recall_fscore_support +from jabs.behavior.postprocessing import PostprocessingPipeline from jabs.core.constants import MULTICLASS_NONE_BEHAVIOR +from jabs.project.track_labels import TrackLabels from . import classifier_utils -from .training_report import BinaryCVResult, CrossValidationResult, MultiClassCVResult +from .cv_postprocessing import ( + FoldPostprocessingEvaluation, + enabled_stage_configs, + evaluate_group_with_postprocessing, +) +from .training_report import ( + BinaryCVResult, + CrossValidationResult, + MultiClassCVResult, + PostprocessedMetrics, +) if TYPE_CHECKING: from jabs.classifier import Classifier, MultiClassClassifier from jabs.project import Project +logger = logging.getLogger(__name__) + +# Binary class labels, in the order the report's metric fields expect. +_BINARY_LABELS = [int(TrackLabels.Label.NOT_BEHAVIOR), int(TrackLabels.Label.BEHAVIOR)] + class CVFeatures(TypedDict): """Feature payload used by cross-validation helper.""" @@ -215,6 +234,143 @@ def _build_multiclass_cv_result( ) +class _PostprocessingEvaluationContext(TypedDict): + """Everything needed to evaluate the postprocessing pipeline for a fold.""" + + pipeline: PostprocessingPipeline + behavior_settings: dict + window_size: int + + +def _postprocessing_context( + project: "Project", + behavior: str, + is_multiclass: bool, + emit_status: Callable[[str], None], +) -> _PostprocessingEvaluationContext | None: + """Build the postprocessing evaluation context, or ``None`` to skip it. + + Evaluation is skipped when the project is in multi-class mode (the + postprocessing pipeline is binary-only) or when the behavior has no enabled + stages, in which case the pipeline would be a no-op and not worth the cost + of re-predicting every held-out track. + """ + if is_multiclass: + logger.warning( + "Postprocessing evaluation was requested but is not supported in " + "multi-class mode; skipping" + ) + emit_status("Postprocessing evaluation is not supported in multi-class mode; skipping") + return None + + config = project.settings_manager.postprocessing_config(behavior) + if not enabled_stage_configs(config): + logger.info( + "Postprocessing evaluation was requested for %s but no stages are enabled; skipping", + behavior, + ) + emit_status("No postprocessing stages are enabled; skipping postprocessing evaluation") + return None + + behavior_settings = project.settings_manager.get_behavior(behavior) + return _PostprocessingEvaluationContext( + pipeline=PostprocessingPipeline(config), + behavior_settings=behavior_settings, + window_size=behavior_settings["window_size"], + ) + + +def _build_postprocessed_metrics( + evaluation: FoldPostprocessingEvaluation, + raw_accuracy: float, +) -> PostprocessedMetrics: + """Score one fold's postprocessed predictions against its ground truth. + + Metrics are computed with an explicit binary label set: postprocessing can + in principle leave a frame with no prediction (``-1``), and letting sklearn + infer the label set from the data would silently shift which array element + belongs to which class. + + Args: + evaluation: Ground truth and predictions for the fold's labeled frames. + raw_accuracy: Accuracy the fold's raw metrics reported, used only for a + consistency check. + + Returns: + The postprocessed metrics for the fold. + """ + # The full-sequence pass predicts the same rows the fold's raw metrics used, + # so its raw accuracy should match. A mismatch means the two paths disagree + # about features or settings, which is worth surfacing. + full_sequence_raw_accuracy = classifier_utils.accuracy_score(evaluation.truth, evaluation.raw) + if not np.isclose(full_sequence_raw_accuracy, raw_accuracy, atol=1e-6): + logger.warning( + "Raw accuracy from the full-sequence postprocessing pass (%.6f) does not match " + "the fold's raw accuracy (%.6f); postprocessed metrics may not be comparable", + full_sequence_raw_accuracy, + raw_accuracy, + ) + + precision, recall, f1, _ = precision_recall_fscore_support( + evaluation.truth, + evaluation.postprocessed, + labels=_BINARY_LABELS, + zero_division=0, + ) + return PostprocessedMetrics( + accuracy=classifier_utils.accuracy_score(evaluation.truth, evaluation.postprocessed), + confusion_matrix=sk_confusion_matrix( + evaluation.truth, evaluation.postprocessed, labels=_BINARY_LABELS + ), + precision_not_behavior=float(precision[0]), + precision_behavior=float(precision[1]), + recall_not_behavior=float(recall[0]), + recall_behavior=float(recall[1]), + f1_behavior=float(f1[1]), + ) + + +def _evaluate_fold_postprocessing( + classifier: "Classifier | MultiClassClassifier", + project: "Project", + behavior: str, + group_info: dict, + context: _PostprocessingEvaluationContext, + raw_accuracy: float, + emit_status: Callable[[str], None], + terminate_callback: Callable[[], None] | None, +) -> PostprocessedMetrics | None: + """Evaluate the postprocessing pipeline for one fold's held-out group. + + Returns: + The postprocessed metrics, or ``None`` when the group has no members + recorded or produced no scorable frames. + """ + members = group_info.get("members") or [] + if not members: + logger.warning( + "Cross-validation group %r has no members recorded; " + "skipping postprocessing evaluation for this fold", + group_info, + ) + return None + + evaluation = evaluate_group_with_postprocessing( + classifier=classifier, + project=project, + behavior=behavior, + members=members, + pipeline=context["pipeline"], + behavior_settings=context["behavior_settings"], + window_size=context["window_size"], + status_callback=emit_status, + terminate_callback=terminate_callback, + ) + if evaluation is None: + return None + return _build_postprocessed_metrics(evaluation, raw_accuracy) + + def run_leave_one_group_out_cv( classifier: "Classifier | MultiClassClassifier", project: "Project", @@ -225,6 +381,7 @@ def run_leave_one_group_out_cv( status_callback: Callable[[str], None] | None = None, progress_callback: Callable[[], None] | None = None, terminate_callback: Callable[[], None] | None = None, + evaluate_postprocessing: bool = False, ) -> list[CrossValidationResult]: """Run leave-one-group-out cross-validation for a classifier. @@ -239,6 +396,11 @@ def run_leave_one_group_out_cv( progress_callback: Optional callback for progress updates (no arguments). terminate_callback: Optional callback to check for early termination (no arguments, should raise if termination is requested). + evaluate_postprocessing: When True, also report metrics with the + behavior's prediction postprocessing pipeline applied. This + re-predicts each held-out group's full tracks (see + :mod:`jabs.classifier.cv_postprocessing`), so it costs roughly one + classification pass over the labeled identities. Binary mode only. Returns: List of cross-validation iteration results. @@ -266,6 +428,14 @@ def emit_progress() -> None: if k == 0: return cv_results + # Built after the k check so a skipped CV run neither reads postprocessing + # settings nor reports on a pipeline it will never apply. + postprocessing_context = ( + _postprocessing_context(project, behavior, is_multiclass, emit_status) + if evaluate_postprocessing + else None + ) + emit_status("Generating train/test splits") data_generator = classifier.leave_one_group_out( features["per_frame"], @@ -309,16 +479,26 @@ def emit_progress() -> None: ) ) else: - cv_results.append( - _build_binary_cv_result( - i + 1, - test_label, - accuracy, - confusion, - top_features, - data, - predictions, - ) + binary_result = _build_binary_cv_result( + i + 1, + test_label, + accuracy, + confusion, + top_features, + data, + predictions, ) + if postprocessing_context is not None: + binary_result.postprocessed = _evaluate_fold_postprocessing( + classifier=classifier, + project=project, + behavior=behavior, + group_info=group_mapping[data["test_group"]], + context=postprocessing_context, + raw_accuracy=accuracy, + emit_status=emit_status, + terminate_callback=terminate_callback, + ) + cv_results.append(binary_result) emit_progress() return cv_results diff --git a/src/jabs/classifier/cv_postprocessing.py b/src/jabs/classifier/cv_postprocessing.py new file mode 100644 index 00000000..81cc425d --- /dev/null +++ b/src/jabs/classifier/cv_postprocessing.py @@ -0,0 +1,201 @@ +"""Postprocessed evaluation of a cross-validation fold. + +Cross-validation normally scores a fold using only the labeled frames of the +held-out group, in the order they happen to be stacked in the feature matrix. +That is fine for raw per-frame metrics, but it cannot be used to evaluate the +prediction postprocessing pipeline: stitching, duration filtering, and gap +interpolation all reason about *contiguous* frames, and the labeled rows are a +sparse, gap-collapsed subset of the video (two labeled bouts thousands of +frames apart end up as adjacent rows). + +So this module re-predicts the held-out group's full tracks the way the +classify path does, applies the pipeline to those full-length prediction +vectors, and only then restricts to the labeled frames where ground truth +exists. That makes the reported numbers reflect what postprocessing actually +does at inference time, including the runs of frames with no prediction that +:class:`~jabs.behavior.postprocessing.stages.GapInterpolationStage` fills. +""" + +from __future__ import annotations + +import logging +from collections import defaultdict +from collections.abc import Callable +from dataclasses import dataclass +from typing import TYPE_CHECKING + +import numpy as np +import numpy.typing as npt + +from jabs.feature_extraction import IdentityFeatures +from jabs.project.track_labels import TrackLabels + +from .inference import predict_identity + +if TYPE_CHECKING: + from jabs.behavior.postprocessing import PostprocessingPipeline + from jabs.pose_estimation import PoseEstimation + from jabs.project import Project + + from .protocols import ClassifierProtocol + +logger = logging.getLogger(__name__) + + +def enabled_stage_configs(config: list[dict]) -> list[dict]: + """Return the stage configurations that are enabled. + + Mirrors the filter :class:`~jabs.behavior.postprocessing.PostprocessingPipeline` + applies when it builds its stages, so callers can tell whether a pipeline + would do anything at all before paying for a full-sequence evaluation. + + Args: + config: Ordered list of stage configuration dicts. + + Returns: + The subset of ``config`` whose stages are enabled, in order. + """ + return [stage for stage in config if stage.get("enabled", True)] + + +@dataclass(frozen=True) +class FoldPostprocessingEvaluation: + """Ground truth and predictions for one fold, restricted to labeled frames. + + All three arrays are the same length and aligned element-wise: they are the + concatenation, over every ``(video, identity)`` in the held-out group, of + that identity's labeled frames. + + Attributes: + truth: Ground-truth labels. + raw: Predictions before postprocessing. + postprocessed: Predictions after the postprocessing pipeline. + """ + + truth: npt.NDArray[np.int8] + raw: npt.NDArray[np.int8] + postprocessed: npt.NDArray[np.int8] + + +def _identity_labels( + project: Project, + video: str, + identity: int, + behavior: str, + pose_est: PoseEstimation, +) -> npt.NDArray[np.int8] | None: + """Load one identity's ground-truth label vector for a behavior. + + Frames where the identity does not exist are forced to + ``TrackLabels.Label.NONE``, mirroring + :func:`~jabs.project.parallel_workers.collect_binary_labeled_features` so + the frames scored here are exactly the fold's test rows. + + Returns: + The per-frame label vector, or ``None`` when the video has no + annotations at all. + """ + labels_obj = project.video_manager.load_video_labels(video, pose_est) + if labels_obj is None: + return None + labels = labels_obj.get_track_labels(str(identity), behavior).get_labels() + identity_mask = pose_est.identity_mask(identity).astype(bool) + labels[~identity_mask] = TrackLabels.Label.NONE + return labels + + +def evaluate_group_with_postprocessing( + classifier: ClassifierProtocol, + project: Project, + behavior: str, + members: list[tuple[str, int]], + pipeline: PostprocessingPipeline, + behavior_settings: dict, + window_size: int, + status_callback: Callable[[str], None] | None = None, + terminate_callback: Callable[[], None] | None = None, +) -> FoldPostprocessingEvaluation | None: + """Re-predict a held-out group's full tracks and apply the postprocessing pipeline. + + Members are processed grouped by video so each pose file is opened once, and + one identity's features are released before the next is loaded - a full-video + feature matrix for a long video is large enough that holding a whole group's + worth at once is not viable. + + Args: + classifier: Classifier trained on this fold's training split. + project: Project providing videos, poses, annotations, and features. + behavior: Behavior being evaluated. + members: ``(video, identity)`` pairs making up the held-out group. + pipeline: Postprocessing pipeline to evaluate. + behavior_settings: Behavior-scoped settings used for feature extraction. + window_size: Window size to use for window features. + status_callback: Optional callback for status updates. + terminate_callback: Optional callback that raises if the caller has + requested early termination. + + Returns: + Ground truth and predictions restricted to labeled frames, or ``None`` + when the group yielded no labeled frames (nothing to score). + """ + truth_parts: list[npt.NDArray[np.int8]] = [] + raw_parts: list[npt.NDArray[np.int8]] = [] + postprocessed_parts: list[npt.NDArray[np.int8]] = [] + + by_video: dict[str, list[int]] = defaultdict(list) + for video, identity in members: + by_video[video].append(identity) + + for video, identities in by_video.items(): + if terminate_callback: + terminate_callback() + pose_est = project.load_pose_est(project.video_manager.video_path(video)) + + for identity in identities: + if terminate_callback: + terminate_callback() + if status_callback: + status_callback(f"Postprocessing evaluation: {video} [{identity}]") + + labels = _identity_labels(project, video, identity, behavior, pose_est) + if labels is None: + logger.warning( + "No annotations found for %s while evaluating postprocessing", video + ) + continue + labeled = labels != TrackLabels.Label.NONE + if not labeled.any(): + continue + + features = IdentityFeatures( + video, + identity, + project.feature_dir, + pose_est, + fps=pose_est.fps, + op_settings=behavior_settings, + cache_format=project.cache_format, + ) + prediction = predict_identity(classifier, features, window_size) + if prediction is None: + logger.warning( + "No features for %s identity %d while evaluating postprocessing", + video, + identity, + ) + continue + + postprocessed = pipeline.run(prediction.predictions, prediction.confidence) + + truth_parts.append(labels[labeled]) + raw_parts.append(prediction.predictions[labeled]) + postprocessed_parts.append(postprocessed[labeled]) + + if not truth_parts: + return None + + return FoldPostprocessingEvaluation( + truth=np.concatenate(truth_parts), + raw=np.concatenate(raw_parts), + postprocessed=np.concatenate(postprocessed_parts), + ) diff --git a/src/jabs/classifier/inference.py b/src/jabs/classifier/inference.py new file mode 100644 index 00000000..8d7d3d51 --- /dev/null +++ b/src/jabs/classifier/inference.py @@ -0,0 +1,84 @@ +"""Full-sequence inference for a single identity. + +This is the inference step shared by prediction and by cross-validation's +postprocessing evaluation. Both need predictions over *every* frame of a +video for one identity - not just the labeled frames - because the +postprocessing pipeline reasons about contiguous bouts and about runs of +frames that have no prediction at all. + +Keeping it in one place means the metrics cross-validation reports for the +postprocessing pipeline are computed from the same predictions the classify +path would produce for the same identity and model. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import TYPE_CHECKING + +import numpy as np +import numpy.typing as npt +import pandas as pd + +if TYPE_CHECKING: + from jabs.feature_extraction import IdentityFeatures + + from .protocols import ClassifierProtocol + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class IdentityPrediction: + """Per-frame prediction output for one identity over a full video. + + All three arrays are indexed by global frame number and span every frame of + the video. Frames where the identity does not exist carry a ``-1`` + prediction and zero probability. + + Attributes: + probabilities: Full per-class probability matrix, shape + ``(n_frames, n_classes)``. + predictions: Predicted class index per frame, ``-1`` where the identity + has no pose. + confidence: Probability of the chosen class per frame. + """ + + probabilities: npt.NDArray[np.float32] + predictions: npt.NDArray[np.int8] + confidence: npt.NDArray[np.float32] + + +def predict_identity( + classifier: ClassifierProtocol, + identity_features: IdentityFeatures, + window_size: int, +) -> IdentityPrediction | None: + """Predict every frame of one identity's track. + + Args: + classifier: Trained classifier used for inference. + identity_features: Feature accessor for the identity being predicted. + window_size: Window size to use for window features. + + Returns: + The per-frame prediction arrays, or ``None`` when the identity has no + feature rows at all (callers substitute a zero-filled result sized to + the video's frame count). + """ + feature_values = identity_features.get_features(window_size) + per_frame_features = pd.DataFrame(feature_values["per_frame"]) + window_features = pd.DataFrame(feature_values["window"]) + data = classifier.combine_data(per_frame_features, window_features) + + if data.shape[0] == 0: + return None + + probabilities = classifier.predict_proba(data, feature_values["frame_indexes"]) + predictions, confidence = classifier.derive_predictions(probabilities) + return IdentityPrediction( + probabilities=probabilities, + predictions=predictions, + confidence=confidence, + ) diff --git a/src/jabs/classifier/protocols.py b/src/jabs/classifier/protocols.py index 8a1b2fd8..e08cb180 100644 --- a/src/jabs/classifier/protocols.py +++ b/src/jabs/classifier/protocols.py @@ -65,6 +65,34 @@ def predict_proba( """ ... + @staticmethod + def combine_data(per_frame: pd.DataFrame, window: pd.DataFrame) -> pd.DataFrame: + """Combine per-frame and window feature DataFrames into one. + + Args: + per_frame: Per-frame feature DataFrame. + window: Window feature DataFrame. + + Returns: + The combined feature DataFrame. + """ + ... + + @staticmethod + def derive_predictions( + probabilities: npt.NDArray[np.floating], + ) -> tuple[npt.NDArray[np.int8], npt.NDArray[np.floating]]: + """Derive class predictions and confidence from class probabilities. + + Args: + probabilities: Array of shape ``(n_frames, n_classes)`` of predicted + class probabilities. + + Returns: + Tuple ``(predictions, confidence)``. + """ + ... + def save(self, path: Path) -> None: """Serialize the classifier to disk. diff --git a/src/jabs/classifier/training_report.py b/src/jabs/classifier/training_report.py index 948ce8ba..8b1c5dcf 100644 --- a/src/jabs/classifier/training_report.py +++ b/src/jabs/classifier/training_report.py @@ -33,6 +33,33 @@ class CrossValidationResult: top_features: list[tuple[str, float]] = field(default_factory=list) +@dataclass +class PostprocessedMetrics: + """Binary metrics for one CV iteration after prediction postprocessing. + + Computed on the same held-out labeled frames as the iteration's raw + metrics, so the two are directly comparable. Support counts are not + repeated here because the ground truth is unchanged. + + Attributes: + accuracy: Classification accuracy (0.0 to 1.0). + confusion_matrix: Confusion matrix, shape ``(2, 2)``. + precision_behavior: Precision for the behavior class. + precision_not_behavior: Precision for the not-behavior class. + recall_behavior: Recall for the behavior class. + recall_not_behavior: Recall for the not-behavior class. + f1_behavior: F1 score for the behavior class. + """ + + accuracy: float = 0.0 + confusion_matrix: np.ndarray | None = None + precision_behavior: float = 0.0 + precision_not_behavior: float = 0.0 + recall_behavior: float = 0.0 + recall_not_behavior: float = 0.0 + f1_behavior: float = 0.0 + + @dataclass class BinaryCVResult(CrossValidationResult): """Binary cross-validation iteration result. @@ -45,6 +72,8 @@ class BinaryCVResult(CrossValidationResult): f1_behavior: F1 score for the behavior class. support_behavior: Number of behavior frames in the test set. support_not_behavior: Number of not-behavior frames in the test set. + postprocessed: Metrics for the same iteration with the postprocessing + pipeline applied, or ``None`` when postprocessing was not evaluated. """ precision_behavior: float = 0.0 @@ -54,6 +83,7 @@ class BinaryCVResult(CrossValidationResult): f1_behavior: float = 0.0 support_behavior: int = 0 support_not_behavior: int = 0 + postprocessed: PostprocessedMetrics | None = None @dataclass @@ -107,6 +137,9 @@ class TrainingReportData: cv_grouping_strategy: Strategy used for cross-validation grouping. cv_grouping_regex: Filename-pattern regex used for grouping. Only set when the grouping strategy is "Filename Pattern". + postprocessing_stages: Enabled postprocessing stage configurations that + were evaluated during cross-validation, or ``None`` when + postprocessing was not evaluated. """ behavior_name: str @@ -127,6 +160,7 @@ class TrainingReportData: class_frame_counts: dict[str, int] | None = None class_bout_counts: dict[str, int] | None = None cv_grouping_regex: str | None = None + postprocessing_stages: list[dict] | None = None def _escape_markdown(text: str) -> str: @@ -142,6 +176,25 @@ def _is_multiclass_cv(cv_results: list[CrossValidationResult]) -> bool: return bool(cv_results) and isinstance(cv_results[0], MultiClassCVResult) +def _format_postprocessing_stages(stages: list[dict]) -> list[str]: + """Return markdown lines describing the evaluated postprocessing stages.""" + lines = ["- **Postprocessing Evaluated in Cross-Validation:** Yes"] + if not stages: + lines.append(" - *No stages enabled*") + return lines + for stage in stages: + params = stage.get("parameters") or {} + rendered = _escape_markdown(", ".join(f"{name}={value}" for name, value in params.items())) + name = _escape_markdown(str(stage.get("stage_name", "unknown"))) + lines.append(f" - {name}" + (f" ({rendered})" if rendered else "")) + return lines + + +def _postprocessed_results(cv_results: list[CrossValidationResult]) -> list[BinaryCVResult]: + """Return the binary CV results that carry postprocessed metrics.""" + return [r for r in cv_results if isinstance(r, BinaryCVResult) and r.postprocessed is not None] + + def _format_label_counts(data: TrainingReportData) -> list[str]: """Return markdown lines for the label-counts section.""" lines: list[str] = [] @@ -181,6 +234,18 @@ def _format_performance_summary(cv_results: list[CrossValidationResult]) -> list f"- **Mean F1 Score (Behavior):** {np.mean(f1_behavior):.4f} " f"(± {np.std(f1_behavior):.4f})" ) + postprocessed = _postprocessed_results(cv_results) + if postprocessed: + pp_accuracies = [r.postprocessed.accuracy for r in postprocessed] + pp_f1 = [r.postprocessed.f1_behavior for r in postprocessed] + lines.append( + f"- **Mean Accuracy (Postprocessed):** {np.mean(pp_accuracies):.4f} " + f"(± {np.std(pp_accuracies):.4f})" + ) + lines.append( + f"- **Mean F1 Score (Behavior, Postprocessed):** {np.mean(pp_f1):.4f} " + f"(± {np.std(pp_f1):.4f})" + ) return lines @@ -233,6 +298,29 @@ def _multiclass_iteration_row(result: MultiClassCVResult) -> list[str | int]: ] +def _postprocessed_iteration_row(result: BinaryCVResult) -> list[str | int]: + """Return a single iteration row for the postprocessed binary CV table.""" + postprocessed = result.postprocessed + if postprocessed is None: # pragma: no cover - callers filter these out + raise ValueError("result has no postprocessed metrics") + return [ + result.iteration, + f"{postprocessed.accuracy:.4f}", + f"{postprocessed.precision_not_behavior:.4f}", + f"{postprocessed.precision_behavior:.4f}", + f"{postprocessed.recall_not_behavior:.4f}", + f"{postprocessed.recall_behavior:.4f}", + f"{postprocessed.f1_behavior:.4f}", + _escape_markdown(result.test_label), + ] + + +def _format_postprocessed_iteration_table(cv_results: list[CrossValidationResult]) -> str: + """Return the markdown iteration table for postprocessed metrics.""" + rows = [_postprocessed_iteration_row(r) for r in _postprocessed_results(cv_results)] + return tabulate(rows, headers=_BINARY_HEADERS, tablefmt="github") + + def _format_iteration_table(cv_results: list[CrossValidationResult]) -> str: """Return the markdown iteration-details table.""" if _is_multiclass_cv(cv_results): @@ -273,6 +361,8 @@ def generate_markdown_report(data: TrainingReportData) -> str: lines.append(f"- **Symmetric Behavior:** {'Yes' if data.symmetric_behavior else 'No'}") lines.append(f"- **Distance Unit:** {data.distance_unit}") lines.append(f"- **Training Time:** {data.training_time_ms / 1000:.2f} seconds") + if data.postprocessing_stages is not None: + lines.extend(_format_postprocessing_stages(data.postprocessing_stages)) lines.append("") lines.append("### Label Counts") @@ -295,6 +385,19 @@ def generate_markdown_report(data: TrainingReportData) -> str: lines.append("") lines.append(_format_iteration_table(data.cv_results)) lines.append("") + + if _postprocessed_results(data.cv_results): + lines.append("### Iteration Details (Postprocessed)") + lines.append("") + lines.append( + "Metrics for the same held-out frames after applying the prediction " + "postprocessing pipeline. Predictions are made over each held-out " + "identity's full track before postprocessing, then scored on the " + "labeled frames only." + ) + lines.append("") + lines.append(_format_postprocessed_iteration_table(data.cv_results)) + lines.append("") else: lines.append("## Cross-Validation") lines.append("") @@ -360,6 +463,16 @@ def _binary_cv_to_dict(result: BinaryCVResult) -> dict: "support_not_behavior": int(result.support_not_behavior), } ) + if result.postprocessed is not None: + payload["postprocessed"] = { + "accuracy": float(result.postprocessed.accuracy), + "confusion_matrix": _to_python_type(result.postprocessed.confusion_matrix), + "precision_behavior": float(result.postprocessed.precision_behavior), + "precision_not_behavior": float(result.postprocessed.precision_not_behavior), + "recall_behavior": float(result.postprocessed.recall_behavior), + "recall_not_behavior": float(result.postprocessed.recall_not_behavior), + "f1_behavior": float(result.postprocessed.f1_behavior), + } return payload @@ -416,6 +529,7 @@ def generate_json_report(data: TrainingReportData) -> dict: "timestamp": timestamp_str, "cv_grouping_strategy": data.cv_grouping_strategy.value, "cv_grouping_regex": data.cv_grouping_regex, + "postprocessing_stages": _to_python_type(data.postprocessing_stages), "frames_behavior": int(data.frames_behavior), "frames_not_behavior": int(data.frames_not_behavior), "bouts_behavior": int(data.bouts_behavior), diff --git a/src/jabs/project/project.py b/src/jabs/project/project.py index fa6a0c25..75fff7e7 100644 --- a/src/jabs/project/project.py +++ b/src/jabs/project/project.py @@ -1238,7 +1238,11 @@ def _assign_cv_group_ids( each group id back to its source. ``INDIVIDUAL``/``VIDEO`` entries are ``{"video": ..., "identity": ...}``; ``FILENAME_PATTERN`` entries are ``{"video": None, "identity": None, "label": , "videos": [...]}`` - where ``videos`` lists the labeled videos in the group. + where ``videos`` lists the labeled videos in the group. Every entry + also carries ``"members"``: the ``(video, identity)`` pairs with + labeled data in that group, which callers need to re-predict a + held-out group's full tracks. A ``VIDEO`` group for a video with no + labeled identities has an empty ``members`` list. Raises: ValueError: If ``grouping_strategy`` is ``FILENAME_PATTERN`` and @@ -1258,18 +1262,23 @@ def _assign_cv_group_ids( key = (v, ident) if key not in key_to_gid: key_to_gid[key] = gid - group_mapping[gid] = {"video": v, "identity": ident} + group_mapping[gid] = { + "video": v, + "identity": ident, + "members": [(v, ident)], + } gid += 1 elif grouping_strategy == CrossValidationGroupingStrategy.VIDEO: video_to_gid: dict[str, int] = {} for v in videos: if v not in video_to_gid: video_to_gid[v] = gid - group_mapping[gid] = {"video": v, "identity": None} + group_mapping[gid] = {"video": v, "identity": None, "members": []} gid += 1 for video_name, ident in all_group_keys: if video_name == v: key_to_gid[(v, ident)] = video_to_gid[v] + group_mapping[video_to_gid[v]]["members"].append((v, ident)) elif grouping_strategy == CrossValidationGroupingStrategy.FILENAME_PATTERN: pattern = compile_grouping_regex(regex or "") label_to_gid: dict[str, int] = {} @@ -1286,10 +1295,12 @@ def _assign_cv_group_ids( "identity": None, "label": label, "videos": [], + "members": [], } gid += 1 group_gid = label_to_gid[label] key_to_gid[(video_name, ident)] = group_gid + group_mapping[group_gid]["members"].append((video_name, ident)) videos_in_group = group_mapping[group_gid]["videos"] if video_name not in videos_in_group: videos_in_group.append(video_name) diff --git a/src/jabs/project/settings_manager.py b/src/jabs/project/settings_manager.py index 2b636bad..a7622f97 100644 --- a/src/jabs/project/settings_manager.py +++ b/src/jabs/project/settings_manager.py @@ -3,7 +3,13 @@ import typing import jabs.feature_extraction as feature_extraction -from jabs.core.constants import CLASSIFIER_MODE_KEY, CV_GROUPING_KEY, CV_GROUPING_REGEX_KEY +from jabs.core.constants import ( + CLASSIFIER_MODE_KEY, + CV_GROUPING_KEY, + CV_GROUPING_REGEX_KEY, + EVALUATE_POSTPROCESSING_IN_CV_KEY, + POSTPROCESSING_KEY, +) from jabs.core.enums.classifier_mode import DEFAULT_CLASSIFIER_MODE, ClassifierMode from jabs.core.enums.cv_grouping import ( DEFAULT_CV_GROUPING_STRATEGY, @@ -259,7 +265,12 @@ def save_behavior(self, behavior: str, data: dict): defaults = self._project_info.get("defaults", {}) all_behavior_data = self._project_info.get("behavior", {}) - merged_data = all_behavior_data.get(behavior, defaults) + # A behavior with no entry yet starts from a *copy* of the project defaults; + # without the copy, merged_data.update() below would write the behavior's + # settings straight into the shared defaults dict. + merged_data = all_behavior_data.get(behavior) + if merged_data is None: + merged_data = dict(defaults) merged_data.update(data) all_behavior_data[behavior] = merged_data @@ -276,6 +287,31 @@ def get_behavior(self, behavior: str) -> dict: """ return self._project_info.get("behavior", {}).get(behavior, {}) + def postprocessing_config(self, behavior: str) -> list[dict]: + """Get the prediction postprocessing stage configuration for a behavior. + + Args: + behavior: Behavior key to read. + + Returns: + Ordered list of stage configuration dicts, suitable for + :class:`~jabs.behavior.postprocessing.PostprocessingPipeline`. Empty + when the behavior has no postprocessing configured. + """ + return self.get_behavior(behavior).get(POSTPROCESSING_KEY, []) + + def evaluate_postprocessing_in_cv(self, behavior: str) -> bool: + """Return whether cross-validation should also report postprocessed metrics. + + Args: + behavior: Behavior key to read. + + Returns: + True if the behavior is configured to evaluate its postprocessing + pipeline during cross-validation. Defaults to False. + """ + return bool(self.get_behavior(behavior).get(EVALUATE_POSTPROCESSING_IN_CV_KEY, False)) + def remove_behavior(self, behavior: str) -> None: """remove behavior from project settings""" try: diff --git a/src/jabs/resources/docs/user_guide/postprocessing.md b/src/jabs/resources/docs/user_guide/postprocessing.md index e192805a..9d09f753 100644 --- a/src/jabs/resources/docs/user_guide/postprocessing.md +++ b/src/jabs/resources/docs/user_guide/postprocessing.md @@ -10,6 +10,19 @@ Postprocessing settings are saved per behavior in the project settings, so diffe Postprocessing is implemented as a pipeline of steps, which are applied in order. The order of steps is currently fixed, but each step can be enabled or disabled independently. Future versions of JABS may allow reordering of steps. +## Evaluating Postprocessing with Cross-Validation + +Postprocessing changes the predictions, so it also changes how well those predictions match your labels. To measure that, enable **Evaluate in Cross-Validation** in the Tools→Prediction Postprocessing dialog. Each time you train that behavior, the training report will then show a second set of cross-validation metrics with the enabled postprocessing steps applied, next to the raw metrics. + +Because postprocessing steps reason about contiguous bouts, the held-out animal's *entire* track is predicted before the steps are applied — the same way predictions are generated when you classify. Metrics are then computed only on the labeled frames, where ground truth exists. This is what makes the comparison meaningful: steps that depend on the gaps between bouts, or on frames with no prediction at all, behave exactly as they would at prediction time. + +A few things to keep in mind: + +- Training is slower with this enabled, because every held-out animal's full track is predicted in addition to being trained on. The added cost is roughly one classification pass over the labeled animals. The first training run after a feature cache is cleared or invalidated is slower still, since the full-track features have to be computed rather than read from the cache. +- Like the postprocessing settings themselves, this option is saved per behavior. +- Postprocessing is only available for binary classifiers, so this option has no effect in multi-class mode. +- If no postprocessing steps are enabled, the evaluation is skipped (the pipeline would not change anything) and the training report says so. + ## Visualizing Postprocessed Predictions After applying postprocessing, JABS allows you to visualize the effects directly in the GUI. When viewing predictions in the Prediction Timeline, you can toggle between raw and postprocessed predictions to see how postprocessing affects the results. diff --git a/src/jabs/scripts/cli/cli.py b/src/jabs/scripts/cli/cli.py index a1643f82..e71c615f 100644 --- a/src/jabs/scripts/cli/cli.py +++ b/src/jabs/scripts/cli/cli.py @@ -360,6 +360,16 @@ def prune(ctx: click.Context, directory: Path, behavior: str | None): "Report format will be determined by extension (.md for Markdown, .json for JSON). " "If not provided, a default filename will be used.", ) +@click.option( + "--postprocessing/--no-postprocessing", + "evaluate_postprocessing", + default=None, + help="Also report cross-validation metrics with the behavior's prediction " + "postprocessing pipeline applied, so raw and postprocessed performance can be " + "compared. This re-predicts each held-out group's full tracks, costing roughly " + "one classification pass over the labeled identities. Binary mode only. " + "Defaults to the behavior's saved project setting.", +) @click.option( "--mlflow", "mlflow_env", @@ -416,6 +426,7 @@ def cross_validation( grouping_pattern: str | None, classifier: str, report_file: Path | None, + evaluate_postprocessing: bool | None, mlflow_env: str | None, mlflow_experiment: str | None, mlflow_tags: tuple[str, ...], @@ -478,6 +489,7 @@ def cross_validation( k, report_file, grouping_regex=grouping_pattern, + evaluate_postprocessing=evaluate_postprocessing, mlflow_enabled=mlflow_enabled, mlflow_env_file=mlflow_env_file, mlflow_experiment=mlflow_experiment, diff --git a/src/jabs/scripts/cli/cross_validation.py b/src/jabs/scripts/cli/cross_validation.py index 1d9cdec1..f7774b91 100644 --- a/src/jabs/scripts/cli/cross_validation.py +++ b/src/jabs/scripts/cli/cross_validation.py @@ -11,6 +11,7 @@ Classifier, MlflowLoggingError, TrainingReportData, + enabled_stage_configs, log_cross_validation_to_mlflow, run_leave_one_group_out_cv, save_training_report, @@ -30,6 +31,7 @@ def run_cross_validation( k: int, report_file: Path | None = None, grouping_regex: str | None = None, + evaluate_postprocessing: bool | None = None, mlflow_enabled: bool = False, mlflow_env_file: Path | None = None, mlflow_experiment: str | None = None, @@ -53,6 +55,10 @@ def run_cross_validation( grouping_regex (str | None): Regular expression used to extract a grouping key from each video filename. Only used when ``grouping_strategy`` is ``FILENAME_PATTERN``. If None, uses the pattern saved in project settings. + evaluate_postprocessing (bool | None): If True, also report metrics with the + behavior's prediction postprocessing pipeline applied. This re-predicts each + held-out group's full tracks, so it costs roughly one classification pass over + the labeled identities. If None, uses the behavior's saved project setting. mlflow_enabled (bool): If True, push the cross-validation results to MLflow after the report is saved. Callers should only enable this when the optional 'mlflow' dependency is installed (the CLI checks this and fails fast with an @@ -94,6 +100,11 @@ def run_cross_validation( classifier = Classifier(classifier=classifier_type, n_jobs=N_JOBS) + # None means "use the behavior's saved setting", matching how the grouping + # strategy and pattern overrides work. + if evaluate_postprocessing is None: + evaluate_postprocessing = project.settings_manager.evaluate_postprocessing_in_cv(behavior) + console = Console() status_message = "Starting cross-validation..." progress = Progress( @@ -138,11 +149,15 @@ def progress_callback(): k=k, status_callback=status_callback, progress_callback=progress_callback, + evaluate_postprocessing=evaluate_postprocessing, ) console.print(f"Cross-validation complete. {len(cv_results)} iterations performed.") # Print Rich table of results if cv_results: + show_postprocessed = any( + getattr(cv, "postprocessed", None) is not None for cv in cv_results + ) table = Table(title="Cross-Validation Results") table.add_column("Iter", justify="center") table.add_column("Accuracy", justify="right") @@ -151,9 +166,13 @@ def progress_callback(): table.add_column("Recall\n(Behavior)", justify="right") table.add_column("Recall\n(Not Behavior)", justify="right") table.add_column("F1 Score", justify="right") + if show_postprocessed: + # placed next to the raw values they should be compared against + table.add_column("Accuracy\n(Postproc.)", justify="right") + table.add_column("F1 Score\n(Postproc.)", justify="right") table.add_column("Test Group", justify="left") for cv in cv_results: - table.add_row( + row = [ str(cv.iteration), f"{cv.accuracy:.3f}", f"{cv.precision_behavior:.3f}", @@ -161,10 +180,21 @@ def progress_callback(): f"{cv.recall_behavior:.3f}", f"{cv.recall_not_behavior:.3f}", f"{cv.f1_behavior:.3f}", - str(cv.test_label), - ) + ] + if show_postprocessed: + postprocessed = getattr(cv, "postprocessed", None) + row.append(f"{postprocessed.accuracy:.3f}" if postprocessed else "-") + row.append(f"{postprocessed.f1_behavior:.3f}" if postprocessed else "-") + row.append(str(cv.test_label)) + table.add_row(*row) console.print(table) + if not show_postprocessed and evaluate_postprocessing: + console.print( + "[yellow]Postprocessing evaluation was requested but produced no " + "metrics (no enabled stages, or no scorable held-out frames).[/yellow]" + ) + # train final model on all data with console.status( "Training final model on all labeled data for feature importance...", spinner="dots" @@ -244,6 +274,11 @@ def progress_callback(): if effective_grouping_strategy == CrossValidationGroupingStrategy.FILENAME_PATTERN else None ), + postprocessing_stages=( + enabled_stage_configs(project.settings_manager.postprocessing_config(behavior)) + if evaluate_postprocessing + else None + ), ) # Save markdown report diff --git a/src/jabs/ui/classification_thread.py b/src/jabs/ui/classification_thread.py index 40ff4edd..b0af1cbe 100644 --- a/src/jabs/ui/classification_thread.py +++ b/src/jabs/ui/classification_thread.py @@ -1,11 +1,11 @@ import time import numpy as np -import pandas as pd from PySide6.QtCore import QThread, Signal from PySide6.QtWidgets import QWidget from jabs.classifier import Classifier, MultiClassClassifier +from jabs.classifier.inference import predict_identity from jabs.core.enums import ClassifierMode from jabs.feature_extraction import DEFAULT_WINDOW_SIZE, IdentityFeatures from jabs.project import Project @@ -151,25 +151,16 @@ def check_termination_requested() -> None: op_settings=project_settings, cache_format=self._project.cache_format, ) - feature_values = features.get_features( - project_settings.get("window_size", DEFAULT_WINDOW_SIZE) + prediction = predict_identity( + self._classifier, + features, + project_settings.get("window_size", DEFAULT_WINDOW_SIZE), ) - - # reformat the data in a single 2D numpy array to pass to the classifier - per_frame_features = pd.DataFrame(feature_values["per_frame"]) - window_features = pd.DataFrame(feature_values["window"]) - data = self._classifier.combine_data(per_frame_features, window_features) - check_termination_requested() - if data.shape[0] > 0: - prob = self._classifier.predict_proba( - data, feature_values["frame_indexes"] - ) - predictions[identity], confidence = self._classifier.derive_predictions( - prob - ) + if prediction is not None: + predictions[identity] = prediction.predictions probabilities[identity] = strategy.probabilities_for_storage( - prob, confidence + prediction.probabilities, prediction.confidence ) else: predictions[identity] = np.full(pose_est.num_frames, -1, dtype=np.int8) diff --git a/src/jabs/ui/classify_strategy.py b/src/jabs/ui/classify_strategy.py index 9f111815..2b5a61a5 100644 --- a/src/jabs/ui/classify_strategy.py +++ b/src/jabs/ui/classify_strategy.py @@ -96,7 +96,7 @@ def __init__( super().__init__(classifier, project, behavior) self._project_settings = project.settings_manager.get_behavior(behavior) self._postprocessing_pipeline = PostprocessingPipeline( - self._project_settings.get("postprocessing", []) + project.settings_manager.postprocessing_config(behavior) ) def project_settings(self) -> dict: diff --git a/src/jabs/ui/settings_dialog/postprocessing_group.py b/src/jabs/ui/settings_dialog/postprocessing_group.py index b1598733..5e002df7 100644 --- a/src/jabs/ui/settings_dialog/postprocessing_group.py +++ b/src/jabs/ui/settings_dialog/postprocessing_group.py @@ -6,6 +6,7 @@ BoutStitchingStage, GapInterpolationStage, ) +from jabs.core.constants import EVALUATE_POSTPROCESSING_IN_CV_KEY from .settings_group import SettingsGroup @@ -226,3 +227,75 @@ def set_values(self, values: dict) -> None: BoutDurationFilterStage.help().kwargs["min_duration"].default, ) ) + + +class PostprocessingEvaluationSettingsGroup(SettingsGroup): + """Settings group controlling postprocessing evaluation during cross-validation.""" + + def __init__(self, parent=None): + """Initialize the postprocessing evaluation settings group.""" + super().__init__("Cross-Validation Evaluation", parent) + + def _create_controls(self) -> None: + """Create the settings controls.""" + self._evaluate_checkbox = QCheckBox("Evaluate postprocessing during cross-validation") + self._evaluate_checkbox.setToolTip( + "Also report cross-validation metrics with the stages above applied, " + "so you can see how they affect classifier performance." + ) + self.add_control_row("Evaluate in Cross-Validation:", self._evaluate_checkbox) + + def _create_documentation(self) -> QLabel: + """Create help documentation for the cross-validation evaluation setting.""" + help_label = QLabel(self) + help_label.setTextFormat(Qt.TextFormat.RichText) + help_label.setWordWrap(True) + help_label.setText( + """ +

Evaluating Postprocessing During Cross-Validation

+ +

When enabled, each cross-validation iteration reports a second set of + metrics with the enabled postprocessing stages applied. The training report + shows both, so you can compare raw classifier performance against + performance after stitching, duration filtering, and interpolation.

+ +

Because the stages reason about contiguous bouts, the held-out animal's + entire track is predicted before the stages are applied - the same way + predictions are generated when you classify. Metrics are then computed only + on the labeled frames, where ground truth exists. This is what makes the + comparison meaningful: filters that depend on gaps between bouts, or on + frames with no prediction at all, behave exactly as they would at + prediction time.

+ +

Note: This makes training slower, because every held-out animal's + full track is predicted in addition to training. The added cost is roughly + one classification pass over the labeled animals. It is slower still the + first time a behavior is trained after a feature cache is cleared or + invalidated, since the full-track features have to be computed rather than + read from the cache.

+ +

Note: Prediction postprocessing is only available for binary + classifiers, so this setting has no effect in multi-class mode.

+ """ + ) + return help_label + + def get_values(self) -> dict: + """ + Get the current cross-validation evaluation setting. + + Returns: + Dictionary with the setting name and its current value. + """ + return {EVALUATE_POSTPROCESSING_IN_CV_KEY: self._evaluate_checkbox.isChecked()} + + def set_values(self, values: dict) -> None: + """ + Set the cross-validation evaluation setting. + + Args: + values: Dictionary with setting names and their desired values. + """ + self._evaluate_checkbox.setChecked( + bool(values.get(EVALUATE_POSTPROCESSING_IN_CV_KEY, False)) + ) diff --git a/src/jabs/ui/settings_dialog/settings_dialog.py b/src/jabs/ui/settings_dialog/settings_dialog.py index bdb14ea1..a630e3c9 100644 --- a/src/jabs/ui/settings_dialog/settings_dialog.py +++ b/src/jabs/ui/settings_dialog/settings_dialog.py @@ -15,7 +15,13 @@ QWidget, ) -from jabs.core.constants import APP_NAME, CLASSIFIER_MODE_KEY, ORG_NAME +from jabs.core.constants import ( + APP_NAME, + CLASSIFIER_MODE_KEY, + EVALUATE_POSTPROCESSING_IN_CV_KEY, + ORG_NAME, + POSTPROCESSING_KEY, +) from jabs.core.enums import ClassifierMode from jabs.project import Project from jabs.project.settings_manager import SettingsManager @@ -27,6 +33,7 @@ from .postprocessing_group import ( DurationStageSettingsGroup, InterpolationStageSettingsGroup, + PostprocessingEvaluationSettingsGroup, StitchingStageSettingsGroup, ) from .session_tracking_group import SessionTrackingSettingsGroup @@ -415,10 +422,21 @@ def __init__( ) def _create_settings_groups(self, parent: QWidget) -> None: - """Create the settings groups for the dialog.""" - self._settings_groups.append(InterpolationStageSettingsGroup(parent)) - self._settings_groups.append(StitchingStageSettingsGroup(parent)) - self._settings_groups.append(DurationStageSettingsGroup(parent)) + """Create the settings groups for the dialog. + + The stage groups are tracked separately from ``self._settings_groups`` + because their order defines the order stages are applied, and they are + saved as an ordered list rather than merged into a flat settings dict. + """ + self._stage_groups: list = [ + InterpolationStageSettingsGroup(parent), + StitchingStageSettingsGroup(parent), + DurationStageSettingsGroup(parent), + ] + self._settings_groups.extend(self._stage_groups) + + self._evaluation_group = PostprocessingEvaluationSettingsGroup(parent) + self._settings_groups.append(self._evaluation_group) def _create_header_widget(self, parent: QWidget) -> QWidget | None: """Create a header widget for the dialog.""" @@ -444,13 +462,17 @@ def _load_settings(self) -> None: """Unlike the ProjectSettingsDialog, load postprocessing settings from the behavior-specific settings.""" # load the settings for the currently selected behavior behavior_settings = self._settings_manager.get_behavior(self._behavior) - settings = behavior_settings.get("postprocessing", []) + settings = behavior_settings.get(POSTPROCESSING_KEY, []) all_settings = {} for stage in settings: all_settings[stage["stage_name"]] = stage + all_settings[EVALUATE_POSTPROCESSING_IN_CV_KEY] = behavior_settings.get( + EVALUATE_POSTPROCESSING_IN_CV_KEY, False + ) + # we pass all settings to each group and each will pick out the settings relevant # to it based on stage_name for group in self._settings_groups: @@ -458,15 +480,19 @@ def _load_settings(self) -> None: def _on_save(self) -> None: """Save postprocessing settings""" + if not self._validate_all_groups(): + return + # Order matters, since it determines the order stages are applied. # To preserve order they are saved as a list of dicts, each dict representing a stage with its # config so that order will be maintained even though the project.json file is sorted by key. - ordered_stages = [] - for group in self._settings_groups: - ordered_stages.append(group.get_values()) + ordered_stages = [group.get_values() for group in self._stage_groups] + + behavior_settings: dict = {POSTPROCESSING_KEY: ordered_stages} + behavior_settings.update(self._evaluation_group.get_values()) # save the postprocessing settings for the current behavior - self._settings_manager.save_behavior(self._behavior, {"postprocessing": ordered_stages}) + self._settings_manager.save_behavior(self._behavior, behavior_settings) self.accept() diff --git a/src/jabs/ui/training_strategy.py b/src/jabs/ui/training_strategy.py index de092003..2c43fc11 100644 --- a/src/jabs/ui/training_strategy.py +++ b/src/jabs/ui/training_strategy.py @@ -99,6 +99,7 @@ def build_report_data( distance_unit: str, settings: dict, cv_grouping_regex: str | None = None, + postprocessing_stages: list[dict] | None = None, ) -> TrainingReportData: """Assemble the ``TrainingReportData`` for the trained model.""" raise NotImplementedError @@ -176,6 +177,7 @@ def build_report_data( distance_unit: str, settings: dict, cv_grouping_regex: str | None = None, + postprocessing_stages: list[dict] | None = None, ) -> TrainingReportData: """Build the binary-mode training report with frame and bout counts. @@ -205,6 +207,7 @@ def build_report_data( window_size=settings["window_size"], cv_grouping_strategy=cv_grouping_strategy, cv_grouping_regex=cv_grouping_regex, + postprocessing_stages=postprocessing_stages, ) def cv_secondary_metric(self, cv_results: list[CrossValidationResult]) -> float | None: @@ -290,11 +293,14 @@ def build_report_data( distance_unit: str, settings: dict, cv_grouping_regex: str | None = None, + postprocessing_stages: list[dict] | None = None, ) -> TrainingReportData: """Build the multi-class training report with per-class frame and bout counts. Frame and bout counts reflect only the videos trained on; rows and videos - excluded from training are filtered out. + excluded from training are filtered out. ``postprocessing_stages`` is + accepted for interface compatibility and ignored, since prediction + postprocessing is binary-only. """ class_names = self._classifier.get_class_names() behavior_names = self._classifier.behavior_names diff --git a/src/jabs/ui/training_thread.py b/src/jabs/ui/training_thread.py index 8ad7d452..60787bae 100644 --- a/src/jabs/ui/training_thread.py +++ b/src/jabs/ui/training_thread.py @@ -12,6 +12,7 @@ save_training_report, ) from jabs.classifier.cross_validation import run_leave_one_group_out_cv +from jabs.classifier.cv_postprocessing import enabled_stage_configs from jabs.core.constants import FINAL_TRAIN_SEED from jabs.core.enums import ClassifierMode, ProjectDistanceUnit from jabs.project import Project @@ -123,6 +124,19 @@ def id_processed() -> None: try: strategy = self._build_strategy() settings = strategy.effective_settings() + settings_manager = self._project.settings_manager + + # Postprocessing evaluation is binary-only; the pipeline has no + # multi-class semantics yet. + evaluate_postprocessing = ( + settings_manager.classifier_mode != ClassifierMode.MULTICLASS + and settings_manager.evaluate_postprocessing_in_cv(self._behavior) + ) + postprocessing_stages = ( + enabled_stage_configs(settings_manager.postprocessing_config(self._behavior)) + if evaluate_postprocessing + else None + ) self.current_status.emit("Extracting Features") features, group_mapping = strategy.collect_features( @@ -141,6 +155,7 @@ def id_processed() -> None: status_callback=self.current_status.emit, progress_callback=id_processed, terminate_callback=check_termination_requested, + evaluate_postprocessing=evaluate_postprocessing, ) self.current_status.emit("Training Classifier") @@ -165,10 +180,11 @@ def id_processed() -> None: final_top_features=final_top_features, elapsed_ms=elapsed_ms, timestamp=datetime.now(), - cv_grouping_strategy=self._project.settings_manager.cv_grouping_strategy, - cv_grouping_regex=self._project.settings_manager.cv_grouping_regex, + cv_grouping_strategy=settings_manager.cv_grouping_strategy, + cv_grouping_regex=settings_manager.cv_grouping_regex, distance_unit=unit, settings=settings, + postprocessing_stages=postprocessing_stages, ) timestamp_str = training_data.timestamp.strftime("%Y%m%d_%H%M%S") diff --git a/tests/classifier/test_cross_validation.py b/tests/classifier/test_cross_validation.py index 58b33391..aabd46a9 100644 --- a/tests/classifier/test_cross_validation.py +++ b/tests/classifier/test_cross_validation.py @@ -1,8 +1,12 @@ """Tests for cross-validation helpers.""" +import logging + import numpy as np import pandas as pd +import pytest +from jabs.classifier import cross_validation from jabs.classifier.cross_validation import run_leave_one_group_out_cv @@ -158,3 +162,239 @@ def test_multiclass_cv_reuses_classifier_settings_without_resetting() -> None: assert classifier.set_project_settings_calls == 0 assert classifier.train_settings == [{"window_size": 123, "balance_labels": True}] + + +class _BinaryCVClassifier: + """Minimal binary test double yielding a single valid LOGO split.""" + + def __init__(self) -> None: + self.train_calls = 0 + + @staticmethod + def get_leave_one_group_out_max(_labels, _groups, _excluded_groups=None) -> int: + return 1 + + @staticmethod + def leave_one_group_out(*_args, **_kwargs): + yield { + "test_group": 1, + "training_idx": np.array([0, 1], dtype=np.intp), + "test_data": pd.DataFrame({"f": [3.0, 4.0]}), + "test_labels": np.array([0, 1], dtype=np.int8), + "test_idx": np.array([2, 3], dtype=np.intp), + "feature_names": ["f"], + } + + def set_project_settings(self, _project, _behavior=None) -> None: + pass + + def train(self, _data) -> None: + self.train_calls += 1 + + @staticmethod + def predict(_test_data): + return np.array([0, 1], dtype=np.int8) + + @staticmethod + def get_feature_importance(limit=10): + return [] + + +def _binary_features() -> dict: + """Return a small binary feature payload with two CV groups.""" + return { + "per_frame": pd.DataFrame({"a": [1.0, 2.0, 3.0, 4.0]}), + "window": pd.DataFrame({"b": [5.0, 6.0, 7.0, 8.0]}), + "labels": np.array([0, 1, 0, 1], dtype=np.int8), + "groups": np.array([0, 0, 1, 1], dtype=np.int32), + } + + +class _FakeSettingsManager: + """Settings manager stand-in exposing only what CV reads.""" + + def __init__(self, postprocessing: list[dict], window_size: int = 5) -> None: + self._postprocessing = postprocessing + self._window_size = window_size + + def postprocessing_config(self, _behavior: str) -> list[dict]: + return list(self._postprocessing) + + def get_behavior(self, _behavior: str) -> dict: + return {"window_size": self._window_size, "postprocessing": self._postprocessing} + + +class _FakeProject: + """Project stand-in carrying a settings manager.""" + + def __init__(self, settings_manager: _FakeSettingsManager) -> None: + self.settings_manager = settings_manager + + @staticmethod + def get_project_defaults() -> dict: + return {"window_size": 5} + + +_STITCH_CONFIG: list[dict] = [ + { + "stage_name": "BoutStitchingStage", + "enabled": True, + "parameters": {"max_stitch_gap": 2}, + } +] + + +def test_postprocessing_evaluation_attaches_metrics(monkeypatch) -> None: + """When enabled, each binary fold carries postprocessed metrics alongside raw.""" + # postprocessing recovers the frame raw got wrong + evaluation = cross_validation.FoldPostprocessingEvaluation( + truth=np.array([0, 1, 1, 1], dtype=np.int8), + raw=np.array([0, 1, 1, 0], dtype=np.int8), + postprocessed=np.array([0, 1, 1, 1], dtype=np.int8), + ) + captured: dict = {} + + def _fake_evaluate(**kwargs): + captured.update(kwargs) + return evaluation + + monkeypatch.setattr(cross_validation, "evaluate_group_with_postprocessing", _fake_evaluate) + + results = run_leave_one_group_out_cv( + classifier=_BinaryCVClassifier(), + project=_FakeProject(_FakeSettingsManager(_STITCH_CONFIG)), + features=_binary_features(), + group_mapping={1: {"video": "v1.avi", "identity": 0, "members": [("v1.avi", 0)]}}, + behavior="Walk", + k=1, + evaluate_postprocessing=True, + ) + + assert len(results) == 1 + postprocessed = results[0].postprocessed + assert postprocessed is not None + assert postprocessed.accuracy == pytest.approx(1.0) + assert postprocessed.recall_behavior == pytest.approx(1.0) + assert postprocessed.confusion_matrix.shape == (2, 2) + # raw metrics are untouched, so the two are directly comparable + assert results[0].accuracy == pytest.approx(1.0) + assert captured["members"] == [("v1.avi", 0)] + assert captured["window_size"] == 5 + + +def test_postprocessing_evaluation_skipped_without_enabled_stages(monkeypatch) -> None: + """No enabled stages means the pipeline is a no-op, so skip the expensive pass.""" + + def _fail(**_kwargs): + raise AssertionError("postprocessing evaluation should not run") + + monkeypatch.setattr(cross_validation, "evaluate_group_with_postprocessing", _fail) + status_messages: list[str] = [] + + results = run_leave_one_group_out_cv( + classifier=_BinaryCVClassifier(), + project=_FakeProject( + _FakeSettingsManager( + [{"stage_name": "BoutStitchingStage", "enabled": False, "parameters": {}}] + ) + ), + features=_binary_features(), + group_mapping={1: {"video": "v1.avi", "identity": 0, "members": [("v1.avi", 0)]}}, + behavior="Walk", + k=1, + status_callback=status_messages.append, + evaluate_postprocessing=True, + ) + + assert results[0].postprocessed is None + assert any("No postprocessing stages are enabled" in msg for msg in status_messages) + + +def test_postprocessing_evaluation_skipped_in_multiclass_mode(monkeypatch) -> None: + """Postprocessing has no multi-class semantics, so the request is refused.""" + + def _fail(**_kwargs): + raise AssertionError("postprocessing evaluation should not run") + + monkeypatch.setattr(cross_validation, "evaluate_group_with_postprocessing", _fail) + features = { + "per_frame": pd.DataFrame({"a": [1.0, 2.0, 3.0, 4.0]}), + "window": pd.DataFrame({"b": [5.0, 6.0, 7.0, 8.0]}), + "groups": np.array([0, 0, 1, 1], dtype=np.int32), + "labels_by_behavior": { + "None": np.array([1, 0, 1, 0], dtype=np.int8), + "Walk": np.array([0, 1, 0, 1], dtype=np.int8), + }, + } + status_messages: list[str] = [] + + run_leave_one_group_out_cv( + classifier=_MultiClassCVClassifier(), + project=_FakeProject(_FakeSettingsManager(_STITCH_CONFIG)), + features=features, + group_mapping={1: {"video": "v1.avi", "identity": 0, "members": [("v1.avi", 0)]}}, + behavior="Walk", + k=1, + status_callback=status_messages.append, + evaluate_postprocessing=True, + ) + + assert any("not supported in multi-class mode" in msg for msg in status_messages) + + +def test_postprocessing_evaluation_skipped_when_group_has_no_members(monkeypatch) -> None: + """A group with no recorded members cannot be re-predicted, so it is skipped.""" + + def _fail(**_kwargs): + raise AssertionError("postprocessing evaluation should not run") + + monkeypatch.setattr(cross_validation, "evaluate_group_with_postprocessing", _fail) + + results = run_leave_one_group_out_cv( + classifier=_BinaryCVClassifier(), + project=_FakeProject(_FakeSettingsManager(_STITCH_CONFIG)), + features=_binary_features(), + group_mapping={1: {"video": "v1.avi", "identity": 0, "members": []}}, + behavior="Walk", + k=1, + evaluate_postprocessing=True, + ) + + assert results[0].postprocessed is None + + +def test_postprocessed_metrics_use_explicit_binary_labels() -> None: + """A leftover -1 prediction must not shift which array element is which class. + + Interpolation can leave a frame with no prediction. Letting sklearn infer + the label set would make -1 the first class and silently relabel the + precision/recall entries. + """ + evaluation = cross_validation.FoldPostprocessingEvaluation( + truth=np.array([0, 0, 1, 1], dtype=np.int8), + raw=np.array([0, 0, 1, 1], dtype=np.int8), + postprocessed=np.array([0, 0, 1, -1], dtype=np.int8), + ) + + metrics = cross_validation._build_postprocessed_metrics(evaluation, raw_accuracy=1.0) + + assert metrics.precision_not_behavior == pytest.approx(1.0) + assert metrics.precision_behavior == pytest.approx(1.0) + # the -1 frame counts as a miss for the behavior class, not as its own class + assert metrics.recall_behavior == pytest.approx(0.5) + assert metrics.accuracy == pytest.approx(0.75) + assert metrics.confusion_matrix.shape == (2, 2) + + +def test_postprocessed_metrics_warn_on_raw_accuracy_mismatch(caplog) -> None: + """A raw-accuracy disagreement between the two paths is surfaced, not hidden.""" + evaluation = cross_validation.FoldPostprocessingEvaluation( + truth=np.array([0, 1], dtype=np.int8), + raw=np.array([1, 0], dtype=np.int8), + postprocessed=np.array([0, 1], dtype=np.int8), + ) + + with caplog.at_level(logging.WARNING, logger="jabs.classifier.cross_validation"): + cross_validation._build_postprocessed_metrics(evaluation, raw_accuracy=1.0) + + assert "does not match" in caplog.text diff --git a/tests/classifier/test_cv_postprocessing.py b/tests/classifier/test_cv_postprocessing.py new file mode 100644 index 00000000..ec3a3b54 --- /dev/null +++ b/tests/classifier/test_cv_postprocessing.py @@ -0,0 +1,317 @@ +"""Tests for postprocessed evaluation of cross-validation folds.""" + +import numpy as np +import pytest + +from jabs.behavior.postprocessing import PostprocessingPipeline +from jabs.behavior.postprocessing.stages import BoutStitchingStage +from jabs.classifier import cv_postprocessing +from jabs.classifier.cv_postprocessing import ( + enabled_stage_configs, + evaluate_group_with_postprocessing, +) +from jabs.classifier.inference import IdentityPrediction +from jabs.project.track_labels import TrackLabels + +NONE = int(TrackLabels.Label.NONE) +NOT_BEHAVIOR = int(TrackLabels.Label.NOT_BEHAVIOR) +BEHAVIOR = int(TrackLabels.Label.BEHAVIOR) + + +class _FakeTrackLabels: + """Stand-in for a per-identity ``TrackLabels`` returning a fixed vector.""" + + def __init__(self, labels: np.ndarray) -> None: + self._labels = labels + + def get_labels(self) -> np.ndarray: + return self._labels.copy() + + +class _FakeVideoLabels: + """Stand-in for ``VideoLabels`` keyed by ``(identity, behavior)``.""" + + def __init__(self, labels_by_identity: dict[str, np.ndarray]) -> None: + self._labels_by_identity = labels_by_identity + self.requested: list[tuple[str, str]] = [] + + def get_track_labels(self, identity: str, behavior: str) -> _FakeTrackLabels: + self.requested.append((identity, behavior)) + return _FakeTrackLabels(self._labels_by_identity[identity]) + + +class _FakePose: + """Pose stand-in where every identity exists on the frames given as valid.""" + + def __init__(self, num_frames: int, valid_by_identity: dict[int, np.ndarray]) -> None: + self.num_frames = num_frames + self.fps = 30 + self._valid_by_identity = valid_by_identity + self.identities = sorted(valid_by_identity) + + def identity_mask(self, identity: int) -> np.ndarray: + return self._valid_by_identity[identity].astype(np.int8) + + +class _FakeProject: + """Minimal Project stand-in for the postprocessing evaluation path.""" + + def __init__( + self, + labels_by_video: dict[str, dict[str, np.ndarray]], + poses: dict[str, _FakePose], + ) -> None: + self.feature_dir = "features" + self.cache_format = "hdf5" + self._poses = poses + self.video_labels = { + video: _FakeVideoLabels(labels) for video, labels in labels_by_video.items() + } + self.opened_poses: list[str] = [] + + project = self + + class _VideoManager: + @staticmethod + def video_path(video: str) -> str: + return video + + @staticmethod + def load_video_labels(video: str, _pose=None): + return project.video_labels.get(video) + + self.video_manager = _VideoManager() + + def load_pose_est(self, video_path: str) -> _FakePose: + """Return the pose stand-in for a video, recording the open.""" + self.opened_poses.append(video_path) + return self._poses[video_path] + + +def _stitching_pipeline(max_stitch_gap: int = 1) -> PostprocessingPipeline: + """Return a pipeline with only the stitching stage enabled.""" + return PostprocessingPipeline( + [ + { + "stage_name": BoutStitchingStage.__name__, + "enabled": True, + "parameters": {"max_stitch_gap": max_stitch_gap}, + } + ] + ) + + +def _patch_prediction(monkeypatch, predictions_by_identity: dict[int, np.ndarray]) -> None: + """Patch feature loading and inference to return fixed full-length predictions.""" + monkeypatch.setattr( + cv_postprocessing, + "IdentityFeatures", + lambda video, identity, *_args, **_kwargs: identity, + ) + + def _fake_predict(_classifier, identity, _window_size) -> IdentityPrediction: + predictions = predictions_by_identity[identity].astype(np.int8) + confidence = np.where(predictions < 0, 0.0, 0.9).astype(np.float32) + probabilities = np.zeros((len(predictions), 2), dtype=np.float32) + return IdentityPrediction( + probabilities=probabilities, + predictions=predictions, + confidence=confidence, + ) + + monkeypatch.setattr(cv_postprocessing, "predict_identity", _fake_predict) + + +@pytest.mark.parametrize( + ("config", "expected_count"), + [ + ([], 0), + ([{"stage_name": "A", "enabled": False, "parameters": {}}], 0), + ([{"stage_name": "A", "enabled": True, "parameters": {}}], 1), + ([{"stage_name": "A", "parameters": {}}], 1), + ], + ids=["empty", "disabled", "enabled", "enabled_by_default"], +) +def test_enabled_stage_configs(config: list[dict], expected_count: int) -> None: + """Only enabled stages count; a missing "enabled" key defaults to enabled.""" + assert len(enabled_stage_configs(config)) == expected_count + + +def test_postprocessing_uses_full_track_not_just_labeled_frames(monkeypatch) -> None: + """Stitching must see the real frame gaps, not the gap-collapsed labeled rows. + + Frames 0-4 and 15-19 are labeled BEHAVIOR; frames 5-14 are unlabeled. The + raw prediction is BEHAVIOR everywhere except a single NOT_BEHAVIOR frame at + 4 and another at 15, each flanked by BEHAVIOR bouts. Over the full track + those are two separate 1-frame gaps and both stitch with + ``max_stitch_gap=1``. If the pipeline instead ran on only the labeled rows + the two frames would be adjacent, forming one 2-frame gap that would not + stitch - so a perfect postprocessed score is what proves full-track + semantics. + """ + num_frames = 20 + labels = np.full(num_frames, NONE, dtype=np.int8) + labels[0:5] = BEHAVIOR + labels[15:20] = BEHAVIOR + + raw = np.full(num_frames, BEHAVIOR, dtype=np.int8) + raw[4] = NOT_BEHAVIOR + raw[15] = NOT_BEHAVIOR + + pose = _FakePose(num_frames, {0: np.ones(num_frames, dtype=bool)}) + project = _FakeProject( + labels_by_video={"video.avi": {"0": labels}}, + poses={"video.avi": pose}, + ) + _patch_prediction(monkeypatch, {0: raw}) + + evaluation = evaluate_group_with_postprocessing( + classifier=object(), + project=project, + behavior="Walk", + members=[("video.avi", 0)], + pipeline=_stitching_pipeline(max_stitch_gap=1), + behavior_settings={"window_size": 5}, + window_size=5, + ) + + assert evaluation is not None + assert len(evaluation.truth) == 10 + assert evaluation.truth.tolist() == [BEHAVIOR] * 10 + # raw got frames 4 and 15 wrong + assert evaluation.raw.tolist() == [1, 1, 1, 1, 0, 0, 1, 1, 1, 1] + # stitching recovered both, which is only possible on the full track + assert evaluation.postprocessed.tolist() == [BEHAVIOR] * 10 + + +def test_evaluation_excludes_frames_where_identity_is_absent(monkeypatch) -> None: + """Labels are forced to NONE where the identity has no pose, matching CV's test rows.""" + num_frames = 6 + labels = np.array([BEHAVIOR] * num_frames, dtype=np.int8) + valid = np.array([True, True, True, False, False, False]) + + pose = _FakePose(num_frames, {0: valid}) + project = _FakeProject( + labels_by_video={"video.avi": {"0": labels}}, + poses={"video.avi": pose}, + ) + raw = np.array([BEHAVIOR, BEHAVIOR, BEHAVIOR, -1, -1, -1], dtype=np.int8) + _patch_prediction(monkeypatch, {0: raw}) + + evaluation = evaluate_group_with_postprocessing( + classifier=object(), + project=project, + behavior="Walk", + members=[("video.avi", 0)], + pipeline=_stitching_pipeline(), + behavior_settings={"window_size": 5}, + window_size=5, + ) + + assert evaluation is not None + assert len(evaluation.truth) == 3 + assert evaluation.raw.tolist() == [BEHAVIOR] * 3 + + +def test_evaluation_spans_all_group_members_and_opens_each_pose_once(monkeypatch) -> None: + """A group with several identities across videos is concatenated, pose files opened once.""" + labels = np.array([BEHAVIOR, NOT_BEHAVIOR], dtype=np.int8) + valid = np.ones(2, dtype=bool) + + project = _FakeProject( + labels_by_video={ + "a.avi": {"0": labels, "1": labels}, + "b.avi": {"0": labels}, + }, + poses={ + "a.avi": _FakePose(2, {0: valid, 1: valid}), + "b.avi": _FakePose(2, {0: valid}), + }, + ) + _patch_prediction(monkeypatch, {0: labels, 1: labels}) + + evaluation = evaluate_group_with_postprocessing( + classifier=object(), + project=project, + behavior="Walk", + members=[("a.avi", 0), ("b.avi", 0), ("a.avi", 1)], + pipeline=_stitching_pipeline(), + behavior_settings={"window_size": 5}, + window_size=5, + ) + + assert evaluation is not None + assert len(evaluation.truth) == 6 + assert project.opened_poses == ["a.avi", "b.avi"] + + +def test_evaluation_returns_none_when_no_labeled_frames(monkeypatch) -> None: + """A group whose members have no labeled frames yields nothing to score.""" + num_frames = 4 + labels = np.full(num_frames, NONE, dtype=np.int8) + project = _FakeProject( + labels_by_video={"video.avi": {"0": labels}}, + poses={"video.avi": _FakePose(num_frames, {0: np.ones(num_frames, dtype=bool)})}, + ) + _patch_prediction(monkeypatch, {0: np.zeros(num_frames, dtype=np.int8)}) + + evaluation = evaluate_group_with_postprocessing( + classifier=object(), + project=project, + behavior="Walk", + members=[("video.avi", 0)], + pipeline=_stitching_pipeline(), + behavior_settings={"window_size": 5}, + window_size=5, + ) + + assert evaluation is None + + +def test_evaluation_skips_videos_without_annotations(monkeypatch) -> None: + """A member whose video has no annotation file is skipped, not fatal.""" + labels = np.array([BEHAVIOR, NOT_BEHAVIOR], dtype=np.int8) + valid = np.ones(2, dtype=bool) + project = _FakeProject( + labels_by_video={"a.avi": {"0": labels}}, + poses={"a.avi": _FakePose(2, {0: valid}), "missing.avi": _FakePose(2, {0: valid})}, + ) + _patch_prediction(monkeypatch, {0: labels}) + + evaluation = evaluate_group_with_postprocessing( + classifier=object(), + project=project, + behavior="Walk", + members=[("a.avi", 0), ("missing.avi", 0)], + pipeline=_stitching_pipeline(), + behavior_settings={"window_size": 5}, + window_size=5, + ) + + assert evaluation is not None + assert len(evaluation.truth) == 2 + + +def test_evaluation_honors_terminate_callback(monkeypatch) -> None: + """The terminate callback is given a chance to abort before each identity.""" + labels = np.array([BEHAVIOR, NOT_BEHAVIOR], dtype=np.int8) + project = _FakeProject( + labels_by_video={"a.avi": {"0": labels}}, + poses={"a.avi": _FakePose(2, {0: np.ones(2, dtype=bool)})}, + ) + _patch_prediction(monkeypatch, {0: labels}) + + def _terminate() -> None: + raise RuntimeError("cancelled") + + with pytest.raises(RuntimeError, match="cancelled"): + evaluate_group_with_postprocessing( + classifier=object(), + project=project, + behavior="Walk", + members=[("a.avi", 0)], + pipeline=_stitching_pipeline(), + behavior_settings={"window_size": 5}, + window_size=5, + terminate_callback=_terminate, + ) diff --git a/tests/classifier/test_inference.py b/tests/classifier/test_inference.py new file mode 100644 index 00000000..aab82e9f --- /dev/null +++ b/tests/classifier/test_inference.py @@ -0,0 +1,82 @@ +"""Tests for full-sequence per-identity inference.""" + +import numpy as np +import pandas as pd + +from jabs.classifier.inference import predict_identity + + +class _FakeIdentityFeatures: + """Stand-in for ``IdentityFeatures`` returning fixed full-video features.""" + + def __init__(self, per_frame: dict, window: dict, frame_indexes: np.ndarray) -> None: + self._features = { + "per_frame": per_frame, + "window": window, + "frame_indexes": frame_indexes, + } + self.requested_window_sizes: list[int] = [] + + def get_features(self, window_size: int) -> dict: + """Record the requested window size and return the fixed features.""" + self.requested_window_sizes.append(window_size) + return self._features + + +class _FakeClassifier: + """Classifier stand-in whose ``predict_proba`` zeroes out excluded frames.""" + + def __init__(self, probabilities: np.ndarray) -> None: + self._probabilities = probabilities + self.predict_proba_calls: list[np.ndarray] = [] + + @staticmethod + def combine_data(per_frame: pd.DataFrame, window: pd.DataFrame) -> pd.DataFrame: + return pd.concat([per_frame, window], axis=1) + + def predict_proba(self, features: pd.DataFrame, frame_indexes=None) -> np.ndarray: + self.predict_proba_calls.append(frame_indexes) + result = np.zeros(self._probabilities.shape, dtype=np.float32) + result[frame_indexes] = self._probabilities[frame_indexes] + return result + + @staticmethod + def derive_predictions(probabilities: np.ndarray): + predictions = np.argmax(probabilities, axis=1).astype(np.int8) + confidence = probabilities[np.arange(len(probabilities)), predictions] + predictions[confidence == 0] = -1 + return predictions, confidence + + +def test_predict_identity_returns_full_length_arrays() -> None: + """Predictions span every frame, with -1 where the identity has no pose.""" + # frame 2 is excluded from frame_indexes, standing in for a missing identity + frame_indexes = np.array([0, 1, 3], dtype=np.intp) + probabilities = np.array([[0.9, 0.1], [0.2, 0.8], [0.5, 0.5], [0.3, 0.7]], dtype=np.float32) + features = _FakeIdentityFeatures( + per_frame={"a": np.arange(4, dtype=np.float64)}, + window={"b": np.arange(4, dtype=np.float64)}, + frame_indexes=frame_indexes, + ) + classifier = _FakeClassifier(probabilities) + + result = predict_identity(classifier, features, window_size=5) + + assert result is not None + assert result.predictions.tolist() == [0, 1, -1, 1] + assert result.confidence[2] == 0.0 + assert result.probabilities.shape == (4, 2) + assert features.requested_window_sizes == [5] + np.testing.assert_array_equal(classifier.predict_proba_calls[0], frame_indexes) + + +def test_predict_identity_returns_none_without_feature_rows() -> None: + """An identity with no feature rows yields None so callers can zero-fill.""" + features = _FakeIdentityFeatures( + per_frame={}, + window={}, + frame_indexes=np.array([], dtype=np.intp), + ) + classifier = _FakeClassifier(np.zeros((0, 2), dtype=np.float32)) + + assert predict_identity(classifier, features, window_size=5) is None diff --git a/tests/classifier/test_training_report.py b/tests/classifier/test_training_report.py index 85dfe050..774b8d49 100644 --- a/tests/classifier/test_training_report.py +++ b/tests/classifier/test_training_report.py @@ -8,6 +8,7 @@ from jabs.classifier.training_report import ( BinaryCVResult, MultiClassCVResult, + PostprocessedMetrics, TrainingReportData, generate_json_report, generate_markdown_report, @@ -478,3 +479,103 @@ def test_multiclass_markdown_empty_count_dicts_do_not_fall_back_to_binary(self): report = generate_markdown_report(data) assert "**Behavior frames:**" not in report assert "**Not-behavior frames:**" not in report + + +class TestPostprocessedReporting: + """Tests for reporting cross-validation metrics with postprocessing applied.""" + + @staticmethod + def _postprocessed_data(sample_training_data, stages: list[dict] | None = None): + """Attach postprocessed metrics to every CV iteration of a report.""" + for offset, result in enumerate(sample_training_data.cv_results): + result.postprocessed = PostprocessedMetrics( + accuracy=0.95 + offset * 0.01, + confusion_matrix=np.array([[190, 10], [8, 142]]), + precision_not_behavior=0.9601, + precision_behavior=0.9702, + recall_not_behavior=0.9803, + recall_behavior=0.9504, + f1_behavior=0.9605, + ) + sample_training_data.postprocessing_stages = ( + stages + if stages is not None + else [ + { + "stage_name": "BoutStitchingStage", + "enabled": True, + "parameters": {"max_stitch_gap": 3}, + } + ] + ) + return sample_training_data + + def test_markdown_contains_postprocessed_table(self, sample_training_data): + """A postprocessed iteration table appears alongside the raw one.""" + data = self._postprocessed_data(sample_training_data) + + report = generate_markdown_report(data) + + assert "### Iteration Details" in report + assert "### Iteration Details (Postprocessed)" in report + # the postprocessed table carries its own metrics, distinct from the raw ones + assert "0.9605" in report + assert "0.9803" in report + # raw metrics survive alongside them + assert "0.9163" in report + + def test_markdown_contains_postprocessed_summary(self, sample_training_data): + """The performance summary reports postprocessed means next to raw means.""" + data = self._postprocessed_data(sample_training_data) + + report = generate_markdown_report(data) + + assert "Mean Accuracy (Postprocessed):" in report + assert "Mean F1 Score (Behavior, Postprocessed):" in report + # raw means are still present, so the two can be compared + assert "**Mean Accuracy:**" in report + + def test_markdown_lists_evaluated_stages(self, sample_training_data): + """The summary records which stages were evaluated, so the report is self-describing.""" + data = self._postprocessed_data(sample_training_data) + + report = generate_markdown_report(data) + + assert "**Postprocessing Evaluated in Cross-Validation:** Yes" in report + assert "BoutStitchingStage" in report + assert "max\\_stitch\\_gap=3" in report # underscores escaped for markdown + + def test_markdown_notes_when_no_stages_enabled(self, sample_training_data): + """Requesting evaluation with no enabled stages is stated rather than silent.""" + sample_training_data.postprocessing_stages = [] + + report = generate_markdown_report(sample_training_data) + + assert "**Postprocessing Evaluated in Cross-Validation:** Yes" in report + assert "*No stages enabled*" in report + + def test_markdown_omits_postprocessing_when_not_evaluated(self, sample_training_data): + """A report for a run without postprocessing evaluation says nothing about it.""" + report = generate_markdown_report(sample_training_data) + + assert "Postprocessing" not in report + assert "(Postprocessed)" not in report + + def test_json_contains_postprocessed_metrics(self, sample_training_data): + """Postprocessed metrics are serialized per iteration plus the stage list.""" + data = self._postprocessed_data(sample_training_data) + + report = generate_json_report(data) + + assert report["postprocessing_stages"][0]["stage_name"] == "BoutStitchingStage" + postprocessed = report["cv_results"][0]["postprocessed"] + assert postprocessed["accuracy"] == pytest.approx(0.95) + assert postprocessed["f1_behavior"] == pytest.approx(0.9605) + assert postprocessed["confusion_matrix"] == [[190, 10], [8, 142]] + + def test_json_omits_postprocessed_when_not_evaluated(self, sample_training_data): + """Iterations without postprocessed metrics carry no postprocessed key.""" + report = generate_json_report(sample_training_data) + + assert report["postprocessing_stages"] is None + assert "postprocessed" not in report["cv_results"][0] diff --git a/tests/project/test_cv_grouping.py b/tests/project/test_cv_grouping.py index 65067907..04f0825d 100644 --- a/tests/project/test_cv_grouping.py +++ b/tests/project/test_cv_grouping.py @@ -43,6 +43,15 @@ def test_assign_cv_group_ids_filename_pattern_groups_videos_by_key() -> None: assert group_mapping[loose_gid]["label"] == "loose.mp4" assert group_mapping[loose_gid]["videos"] == ["loose.mp4"] + # Members name the (video, identity) pairs, which "videos" alone cannot: a + # postprocessing evaluation has to re-predict each held-out identity. + assert group_mapping[cage1_gid]["members"] == [ + ("cage_1_a.mp4", 0), + ("cage_1_a.mp4", 1), + ("cage_1_b.mp4", 0), + ] + assert group_mapping[loose_gid]["members"] == [("loose.mp4", 0)] + def test_assign_cv_group_ids_filename_pattern_ids_are_contiguous() -> None: """Group ids are assigned contiguously from zero in row order.""" @@ -87,6 +96,7 @@ def test_assign_cv_group_ids_video_grouping_unchanged() -> None: assert group_mapping[key_to_gid[("video_a.mp4", 0)]] == { "video": "video_a.mp4", "identity": None, + "members": [("video_a.mp4", 0), ("video_a.mp4", 1)], } @@ -103,4 +113,5 @@ def test_assign_cv_group_ids_individual_grouping_unchanged() -> None: assert group_mapping[key_to_gid[("video_a.mp4", 1)]] == { "video": "video_a.mp4", "identity": 1, + "members": [("video_a.mp4", 1)], } diff --git a/tests/project/test_settings_manager.py b/tests/project/test_settings_manager.py index bc39542b..6f48a498 100644 --- a/tests/project/test_settings_manager.py +++ b/tests/project/test_settings_manager.py @@ -252,3 +252,70 @@ def test_cv_grouping_regex_reads_configured_value(mock_project): settings_manager = SettingsManager(mock_project.project_paths) assert settings_manager.cv_grouping_regex == r"cage_(\d+)" + + +def _write_behavior_settings(mock_project, behavior: str, behavior_settings: dict) -> None: + """Write a project file containing settings for a single behavior.""" + with mock_project.project_paths.project_file.open("w") as f: + json.dump({"behavior": {behavior: behavior_settings}}, f) + + +def test_postprocessing_config_returns_stage_list(mock_project): + """The stage list is returned in the order it was saved.""" + stages = [ + {"stage_name": "GapInterpolationStage", "enabled": False, "parameters": {}}, + {"stage_name": "BoutStitchingStage", "enabled": True, "parameters": {"max_stitch_gap": 3}}, + ] + _write_behavior_settings(mock_project, "Walking", {"postprocessing": stages}) + + settings_manager = SettingsManager(mock_project.project_paths) + + assert settings_manager.postprocessing_config("Walking") == stages + + +def test_postprocessing_config_defaults_to_empty(mock_project): + """A behavior with no postprocessing configured yields an empty list.""" + _write_behavior_settings(mock_project, "Walking", {"window_size": 5}) + + settings_manager = SettingsManager(mock_project.project_paths) + + assert settings_manager.postprocessing_config("Walking") == [] + assert settings_manager.postprocessing_config("Unknown") == [] + + +@pytest.mark.parametrize( + ("stored", "expected"), + [({"evaluate_postprocessing_in_cv": True}, True), ({}, False)], + ids=["enabled", "default"], +) +def test_evaluate_postprocessing_in_cv(mock_project, stored: dict, expected: bool): + """The cross-validation evaluation flag reads back, defaulting to off.""" + _write_behavior_settings(mock_project, "Walking", stored) + + settings_manager = SettingsManager(mock_project.project_paths) + + assert settings_manager.evaluate_postprocessing_in_cv("Walking") is expected + + +def test_save_behavior_for_new_behavior_does_not_mutate_defaults(mock_project): + """Saving settings for a not-yet-present behavior must not rewrite project defaults.""" + with mock_project.project_paths.project_file.open("w") as f: + json.dump({"defaults": {"window_size": 5}, "behavior": {}}, f) + + settings_manager = SettingsManager(mock_project.project_paths) + settings_manager.save_behavior( + "NewBehavior", + { + "postprocessing": [{"stage_name": "BoutStitchingStage", "parameters": {}}], + "evaluate_postprocessing_in_cv": True, + }, + ) + + # the new behavior inherits the defaults plus its own settings + behavior_settings = settings_manager.get_behavior("NewBehavior") + assert behavior_settings["window_size"] == 5 + assert behavior_settings["evaluate_postprocessing_in_cv"] is True + + # ...but the defaults themselves are untouched, so the next new behavior + # does not silently inherit this one's postprocessing configuration + assert settings_manager.project_settings["defaults"] == {"window_size": 5} diff --git a/tests/scripts/test_cross_validation_cli.py b/tests/scripts/test_cross_validation_cli.py index 016a4f80..c0c2e7a6 100644 --- a/tests/scripts/test_cross_validation_cli.py +++ b/tests/scripts/test_cross_validation_cli.py @@ -97,6 +97,29 @@ def test_no_strategy_defaults_to_none(tmp_path: Path, run_cv_spy: mock.Mock) -> assert run_cv_spy.call_args.kwargs["grouping_regex"] is None +@pytest.mark.parametrize( + ("flag", "expected"), + [("--postprocessing", True), ("--no-postprocessing", False)], + ids=["enabled", "disabled"], +) +def test_postprocessing_flag_forwarded( + tmp_path: Path, run_cv_spy: mock.Mock, flag: str, expected: bool +) -> None: + """``--postprocessing``/``--no-postprocessing`` overrides the saved project setting.""" + result = _invoke(tmp_path, flag) + + assert result.exit_code == 0, result.output + assert run_cv_spy.call_args.kwargs["evaluate_postprocessing"] is expected + + +def test_postprocessing_defaults_to_project_setting(tmp_path: Path, run_cv_spy: mock.Mock) -> None: + """Omitting the flag passes None so the behavior's saved setting is used.""" + result = _invoke(tmp_path) + + assert result.exit_code == 0, result.output + assert run_cv_spy.call_args.kwargs["evaluate_postprocessing"] is None + + def test_invalid_grouping_strategy_rejected(tmp_path: Path, run_cv_spy: mock.Mock) -> None: """An unknown strategy is rejected by Click before run_cross_validation is called.""" result = _invoke(tmp_path, "--grouping-strategy", "bogus") diff --git a/tests/ui/_fakes.py b/tests/ui/_fakes.py index 95059381..b1161ad5 100644 --- a/tests/ui/_fakes.py +++ b/tests/ui/_fakes.py @@ -159,6 +159,8 @@ def __init__( cv_grouping_regex="", get_behavior=lambda _behavior: dict(self._DEFAULT_BEHAVIOR_SETTINGS), is_video_excluded=lambda _video: False, + postprocessing_config=lambda _behavior: [], + evaluate_postprocessing_in_cv=lambda _behavior: False, ) self._binary_features = binary_features self._multiclass_features = multiclass_features @@ -216,6 +218,8 @@ def __init__(self, mode: ClassifierMode) -> None: self.settings_manager = SimpleNamespace( classifier_mode=mode, get_behavior=lambda _behavior: {"window_size": 5, "postprocessing": []}, + postprocessing_config=lambda _behavior: [], + evaluate_postprocessing_in_cv=lambda _behavior: False, ) self.feature_manager = SimpleNamespace(distance_unit=ProjectDistanceUnit.PIXEL) self.video_manager = SimpleNamespace( diff --git a/tests/ui/test_settings_dialog.py b/tests/ui/test_settings_dialog.py index 5338e644..8607fc03 100644 --- a/tests/ui/test_settings_dialog.py +++ b/tests/ui/test_settings_dialog.py @@ -3,7 +3,13 @@ import pytest -from jabs.core.constants import CLASSIFIER_MODE_KEY, CV_GROUPING_KEY, CV_GROUPING_REGEX_KEY +from jabs.core.constants import ( + CLASSIFIER_MODE_KEY, + CV_GROUPING_KEY, + CV_GROUPING_REGEX_KEY, + EVALUATE_POSTPROCESSING_IN_CV_KEY, + POSTPROCESSING_KEY, +) from jabs.core.enums import ClassifierMode, CrossValidationGroupingStrategy try: @@ -17,7 +23,13 @@ from jabs.ui.settings_dialog.cross_validation_settings_group import ( CrossValidationSettingsGroup, ) - from jabs.ui.settings_dialog.settings_dialog import _OverlapCheckThread + from jabs.ui.settings_dialog.postprocessing_group import ( + PostprocessingEvaluationSettingsGroup, + ) + from jabs.ui.settings_dialog.settings_dialog import ( + PostprocessingSettingsDialog, + _OverlapCheckThread, + ) SKIP_UI_TESTS = False SKIP_REASON = None @@ -245,3 +257,84 @@ def test_collapsible_section_toggles_disclosure_icon() -> None: section.set_expanded(False) assert section._toggle_btn.icon().cacheKey() == collapsed_key + + +class _FakePostprocessingSettingsManager: + """Settings manager stand-in recording what the postprocessing dialog saves.""" + + def __init__(self, behavior_settings: dict | None = None) -> None: + self._behavior_settings = behavior_settings or {} + self.saved: list[tuple[str, dict]] = [] + + def get_behavior(self, _behavior: str) -> dict: + return dict(self._behavior_settings) + + def save_behavior(self, behavior: str, data: dict) -> None: + self.saved.append((behavior, data)) + + +def test_postprocessing_evaluation_group_roundtrips_flag() -> None: + """The cross-validation evaluation flag round-trips, defaulting to off.""" + group = PostprocessingEvaluationSettingsGroup() + + assert group.get_values() == {EVALUATE_POSTPROCESSING_IN_CV_KEY: False} + + group.set_values({EVALUATE_POSTPROCESSING_IN_CV_KEY: True}) + assert group.get_values() == {EVALUATE_POSTPROCESSING_IN_CV_KEY: True} + + group.set_values({}) + assert group.get_values() == {EVALUATE_POSTPROCESSING_IN_CV_KEY: False} + + +def test_postprocessing_dialog_saves_stages_and_flag_separately() -> None: + """The evaluation flag is a sibling key, not an entry in the ordered stage list. + + The stage list's order defines the order stages are applied, so a non-stage + group must not be swept into it. + """ + settings_manager = _FakePostprocessingSettingsManager() + dialog = PostprocessingSettingsDialog(settings_manager, behavior="Walk") + + dialog._evaluation_group.set_values({EVALUATE_POSTPROCESSING_IN_CV_KEY: True}) + dialog._on_save() + + assert len(settings_manager.saved) == 1 + behavior, data = settings_manager.saved[0] + assert behavior == "Walk" + assert data[EVALUATE_POSTPROCESSING_IN_CV_KEY] is True + + stages = data[POSTPROCESSING_KEY] + assert len(stages) == 3 + assert [stage["stage_name"] for stage in stages] == [ + "GapInterpolationStage", + "BoutStitchingStage", + "BoutDurationFilterStage", + ] + assert all("stage_name" in stage for stage in stages) + + +def test_postprocessing_dialog_loads_saved_flag() -> None: + """An enabled flag stored under the behavior is reflected in the dialog.""" + settings_manager = _FakePostprocessingSettingsManager( + { + EVALUATE_POSTPROCESSING_IN_CV_KEY: True, + POSTPROCESSING_KEY: [ + { + "stage_name": "BoutStitchingStage", + "enabled": True, + "parameters": {"max_stitch_gap": 7}, + } + ], + } + ) + + dialog = PostprocessingSettingsDialog(settings_manager, behavior="Walk") + + assert dialog._evaluation_group.get_values() == {EVALUATE_POSTPROCESSING_IN_CV_KEY: True} + stitching = next( + stage + for stage in (g.get_values() for g in dialog._stage_groups) + if stage["stage_name"] == "BoutStitchingStage" + ) + assert stitching["enabled"] is True + assert stitching["parameters"]["max_stitch_gap"] == 7 diff --git a/tests/ui/test_training_thread.py b/tests/ui/test_training_thread.py index 3280edba..0d257107 100644 --- a/tests/ui/test_training_thread.py +++ b/tests/ui/test_training_thread.py @@ -123,3 +123,110 @@ def _fake_cv(**kwargs): assert reports == ["report"] assert len(completions) == 1 project.session_tracker.classifier_trained.assert_called_once_with("Walk", "catboost", 0) + + +def _capture_cv_kwargs(monkeypatch) -> dict: + """Patch the training thread's CV call and report writers, returning captured kwargs.""" + captured: dict = {} + + def _fake_cv(**kwargs): + captured.update(kwargs) + return [] + + monkeypatch.setattr("jabs.ui.training_thread.run_leave_one_group_out_cv", _fake_cv) + monkeypatch.setattr( + "jabs.ui.training_thread.save_training_report", lambda *_args, **_kwargs: None + ) + monkeypatch.setattr( + "jabs.ui.training_thread.generate_markdown_report", lambda *_args, **_kwargs: "report" + ) + return captured + + +def _binary_features() -> dict: + """Return a minimal binary feature payload.""" + return { + "per_frame": pd.DataFrame({"feat_a": [1.0, 2.0]}), + "window": pd.DataFrame({"feat_b": [3.0, 4.0]}), + "labels": np.array([1, 0], dtype=np.int8), + "groups": np.array([0, 1], dtype=np.int32), + } + + +_STITCH_STAGE = { + "stage_name": "BoutStitchingStage", + "enabled": True, + "parameters": {"max_stitch_gap": 3}, +} + + +def test_training_thread_forwards_postprocessing_setting(monkeypatch, tmp_path) -> None: + """The behavior's saved flag reaches CV, and the stages reach the report.""" + project = FakeTrainingProject( + tmp_path, ClassifierMode.BINARY, binary_features=_binary_features() + ) + project.settings_manager.evaluate_postprocessing_in_cv = lambda _behavior: True + project.settings_manager.postprocessing_config = lambda _behavior: [_STITCH_STAGE] + + captured = _capture_cv_kwargs(monkeypatch) + report_data: list = [] + monkeypatch.setattr( + "jabs.ui.training_thread.generate_markdown_report", + lambda data: report_data.append(data) or "report", + ) + + errors: list[Exception] = [] + thread = TrainingThread(FakeTrainingClassifier(), project, "Walk", (3, 4), k=1) + thread.error_callback.connect(errors.append) + + thread.run() + + assert errors == [] + assert captured["evaluate_postprocessing"] is True + assert report_data[0].postprocessing_stages == [_STITCH_STAGE] + + +def test_training_thread_omits_postprocessing_when_disabled(monkeypatch, tmp_path) -> None: + """With the flag off, CV is not asked to evaluate and the report stays silent.""" + project = FakeTrainingProject( + tmp_path, ClassifierMode.BINARY, binary_features=_binary_features() + ) + captured = _capture_cv_kwargs(monkeypatch) + report_data: list = [] + monkeypatch.setattr( + "jabs.ui.training_thread.generate_markdown_report", + lambda data: report_data.append(data) or "report", + ) + + thread = TrainingThread(FakeTrainingClassifier(), project, "Walk", (3, 4), k=1) + thread.run() + + assert captured["evaluate_postprocessing"] is False + assert report_data[0].postprocessing_stages is None + + +def test_training_thread_skips_postprocessing_in_multiclass_mode(monkeypatch, tmp_path) -> None: + """Postprocessing is binary-only, so multi-class training never requests it.""" + features = { + "per_frame": pd.DataFrame({"feat_a": [1.0, 2.0]}), + "window": pd.DataFrame({"feat_b": [3.0, 4.0]}), + "labels_by_behavior": {"Walk": np.array([1, 0], dtype=np.int8)}, + "groups": np.array([0, 1], dtype=np.int32), + } + project = FakeTrainingProject( + tmp_path, ClassifierMode.MULTICLASS, multiclass_features=features + ) + # even with the flag set, multi-class must not ask for the evaluation + project.settings_manager.evaluate_postprocessing_in_cv = lambda _behavior: True + project.settings_manager.postprocessing_config = lambda _behavior: [_STITCH_STAGE] + + captured = _capture_cv_kwargs(monkeypatch) + + errors: list[Exception] = [] + thread = TrainingThread(FakeTrainingClassifier(), project, "Walk", (3, 4), k=1) + thread.error_callback.connect(errors.append) + + thread.run() + + assert errors == [] + assert captured["evaluate_postprocessing"] is False