Add shared Decision model support for Foundry evaluators

sdk/evaluation/azure-ai-evaluation/CHANGELOG.md

@@ -2,9 +2,19 @@
 
 ## 1.18.10 (Unreleased)
 
+### Features Added
+
+- Added shared model runners and evaluation strategies for asset evaluators using decision models in Microsoft Foundry.
+- Decision evaluator results expose known final-response token usage and model fields in standard evaluation rows, including skipped evaluations, while retaining known metadata on failures. Usage does not include earlier retry attempts.
+
+### Breaking Changes
+
 ### Bugs Fixed
 
 - Hardened local Prompty image resolution so only relative files within the Prompty directory are inlined, with opened-file validation to prevent path changes from redirecting reads and without exposing the local directory in fallback logs.
+- Output Quality and Tool Use Quality asset evaluators retain their aggregate and member metrics when used with `evaluate` without catalog metadata.
+
+### Other Changes
 
 ## 1.18.9 (2026-10-12)
 

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_constants.py

@@ -136,6 +136,23 @@ class _EvaluatorMetricMapping:
         "tool_call_success": ["tool_call_success"],
         "tool_selection": ["tool_selection"],
         "tool_output_utilization": ["tool_output_utilization"],
+        "output_quality": [
+            "output_quality",
+            "fluency",
+            "coherence",
+            "intent_resolution",
+            "task_adherence",
+            "groundedness",
+            "task_completion",
+        ],
+        "tool_use_quality": [
+            "tool_use_quality",
+            "tool_call_accuracy",
+            "tool_call_success",
+            "tool_input_accuracy",
+            "tool_output_utilization",
+            "tool_selection",
+        ],
         "task_navigation_efficiency": ["task_navigation_efficiency"],
         "text_similarity": ["similarity"],
         "string_check": ["string_check"],
@@ -161,6 +178,7 @@ class _EvaluatorMetricMapping:
         "IndirectAttackEvaluator": "indirect_attack",
         "IntentResolutionEvaluator": "intent_resolution",
         "MeteorScoreEvaluator": "meteor_score",
+        "OutputQualityEvaluator": "output_quality",
         "ProtectedMaterialEvaluator": "protected_material",
         "QAEvaluator": "qa",
         "RelevanceEvaluator": "relevance",
@@ -173,6 +191,7 @@ class _EvaluatorMetricMapping:
         "TaskAdherenceEvaluator": "task_adherence",
         "TaskCompletionEvaluator": "task_completion",
         "ToolCallAccuracyEvaluator": "tool_call_accuracy",
+        "ToolUseQualityEvaluator": "tool_use_quality",
         "UngroundedAttributesEvaluator": "ungrounded_attributes",
         "ViolenceEvaluator": "violence",
     }

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py

@@ -7,6 +7,7 @@
 import pandas as pd
 import sys
 import itertools
+import math
 from collections import defaultdict
 from concurrent.futures import Future
 from os import PathLike
@@ -21,13 +22,45 @@
 from ..._legacy._common._thread_pool_executor_with_context import ThreadPoolExecutorWithContext
 from ..._evaluate._utils import _has_aggregator
 from ..._constants import Prefixes, PF_BATCH_TIMEOUT_SEC
+from ..._exceptions import EvaluationException
 
 from .._utils import get_int_env_var as get_int
 
 LOGGER = logging.getLogger("run")
 MISSING_VALUE: Final[int] = sys.maxsize
 
 
+def _decision_error_metadata(exception: EvaluationException) -> Dict[str, Any]:
+    """Keep only known final-response usage and validated provenance for failed Decision rows."""
+    properties: Dict[str, Any] = {}
+    semantic_properties = exception.properties or {}
+    for key in ("prompt_tokens", "completion_tokens", "total_tokens"):
+        value = semantic_properties.get(key)
+        if isinstance(value, int) and not isinstance(value, bool) and value >= 0:
+            properties[key] = value
+    model = semantic_properties.get("model")
+    if isinstance(model, str) and model.strip():
+        properties["model"] = model
+    native = semantic_properties.get("decision_model")
+    if isinstance(native, dict):
+        provenance = {
+            key: native[key]
+            for key in ("deployment", "model", "rubric_hash", "status_rubric_hash", "reason_type")
+            if isinstance(native.get(key), str)
+        }
+        for key in ("status_completion_probability", "status_probability_threshold"):
+            value = native.get(key)
+            if (
+                isinstance(value, (int, float))
+                and not isinstance(value, bool)
+                and math.isfinite(value)
+                and 0 <= value <= 1
+            ):
+                provenance[key] = value
+        properties["decision_model"] = provenance
+    return {"code": exception.category.value, "blame": exception.blame.value, "properties": properties}
+
+
 class RunSubmitterClient:
     def __init__(self, *, raise_on_errors: bool = False, config: Optional[BatchEngineConfig] = None) -> None:
         if config:
@@ -154,12 +187,20 @@ def get_run_summary(self, client_run: BatchClientRun) -> Dict[str, Any]:
 
         # Collect per-line error messages for failed lines
         per_line_errors: Dict[int, str] = {}
+        per_line_error_metadata: Dict[int, Dict[str, Any]] = {}
         if run.result and run.result.details:
             for detail in run.result.details:
                 if detail.error and detail.error.exception:
                     per_line_errors[detail.index] = str(detail.error.exception)
-
-        return {
+                    exception = detail.error.exception
+                    if (
+                        isinstance(exception, EvaluationException)
+                        and isinstance(exception.properties, dict)
+                        and isinstance(exception.properties.get("decision_model"), dict)
+                    ):
+                        per_line_error_metadata[detail.index] = _decision_error_metadata(exception)
+
+        summary = {
             "status": run.status.value,
             "duration": str(run.duration),
             "completed_lines": total_lines - failed_lines,
@@ -177,6 +218,9 @@ def get_run_summary(self, client_run: BatchClientRun) -> Dict[str, Any]:
                 else None
             ),
         }
+        if per_line_error_metadata:
+            summary["per_line_error_metadata"] = per_line_error_metadata
+        return summary
 
     @staticmethod
     def _get_run(run: BatchClientRun) -> Run:

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py

@@ -3301,6 +3301,8 @@ def _add_error_summaries(
     for criteria that failed during evaluation, ensuring proper error reporting.
     When per-line errors are available, only the error for the current row is used;
     rows that succeeded are not stamped with another row's error.
+    Decision-only per-line metadata preserves known usage/provenance in ``properties`` and the
+    existing ``sample`` schema, without treating failed attempts as completed scoring results.
 
     :param run_output_results: List to append error result objects to
     :type run_output_results: List[Dict[str, Any]]
@@ -3357,13 +3359,17 @@ def _add_error_summaries(
         # Use per-line error if available, otherwise fall back to batch-level error
         per_line_errors = criteria_summary.get("per_line_errors", {})
         per_line_error_msg = per_line_errors.get(row_idx, None)
+        decision_error_metadata = criteria_summary.get("per_line_error_metadata", {}).get(row_idx)
 
         if per_line_error_msg is not None:
             # This row has a specific error
             error_info = {
                 "code": criteria_summary.get("error_code"),
                 "message": per_line_error_msg,
             }
+            if decision_error_metadata is not None:
+                error_info["code"] = decision_error_metadata["code"]
+                error_info["blame"] = decision_error_metadata["blame"]
         elif per_line_errors:
             # Per-line errors exist but not for this row — this row succeeded, skip error stamping
             continue
@@ -3378,6 +3384,12 @@ def _add_error_summaries(
 
         metrics = testing_criteria_metadata.get(criteria_name, {}).get("metrics", [])
         criteria_type = testing_criteria_metadata.get(criteria_name, {}).get("type", "azure_ai_evaluator")
+        if decision_error_metadata is not None:
+            evaluator_name = testing_criteria_metadata.get(criteria_name, {}).get("evaluator_name")
+            if isinstance(evaluator_name, str):
+                primary_metric = evaluator_name.removeprefix("builtin.")
+                metrics = sorted(metrics, key=lambda metric: metric != primary_metric)
+        decision_usage_added = False
 
         for metric in metrics:
             if not _should_add_error_summary(run_output_results, criteria_name, metric):
@@ -3395,6 +3407,22 @@ def _add_error_summaries(
                 "status": "error",
                 "sample": sample,
             }
+            if decision_error_metadata is not None:
+                properties = decision_error_metadata["properties"]
+                error_result["properties"] = properties
+                # One failed evaluator invocation may produce several metric errors, but only one billed usage.
+                error_result["sample"] = {"error": error_info}
+                usage = {
+                    key: properties[key]
+                    for key in ("prompt_tokens", "completion_tokens", "total_tokens")
+                    if key in properties
+                }
+                if not decision_usage_added:
+                    if usage:
+                        error_result["sample"]["usage"] = usage
+                    if "model" in properties:
+                        error_result["sample"]["model"] = properties["model"]
+                    decision_usage_added = True
             run_output_results.append(error_result)
 
 

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/__init__.py

@@ -3,13 +3,15 @@
 # ---------------------------------------------------------
 
 from ._base_eval import EvaluatorBase
+from ._base_model_eval import ModelEvaluatorBase
 from ._base_prompty_eval import PromptyEvaluatorBase
 from ._base_rai_svc_eval import RaiServiceEvaluatorBase
 from ._base_multi_eval import MultiEvaluatorBase
 from ._conversation_input import hoist_messages_to_conversation
 
 __all__ = [
     "EvaluatorBase",
+    "ModelEvaluatorBase",
     "PromptyEvaluatorBase",
     "RaiServiceEvaluatorBase",
     "MultiEvaluatorBase",