From 2e6f0952fa8f77435a216f7b8460e30d00d858f9 Mon Sep 17 00:00:00 2001 From: Salma Elshafey Date: Mon, 12 Oct 2026 11:47:03 +0300 Subject: [PATCH 01/24] add support for evaluation with decision models v0 --- .../azure-ai-evaluation/CHANGELOG.md | 2 + .../_evaluators/_common/__init__.py | 2 + .../_evaluators/_common/_base_model_eval.py | 328 +++++++++++ .../_evaluators/_common/_base_prompty_eval.py | 344 +---------- .../_evaluators/_common/_decision_strategy.py | 189 ++++++ .../_evaluators/_common/_model_runners.py | 318 ++++++++++ .../_evaluators/_common/_prompty_strategy.py | 135 +++++ .../unittests/test_decision_model_support.py | 546 ++++++++++++++++++ .../tests/unittests/test_prompty_strategy.py | 134 +++++ 9 files changed, 1673 insertions(+), 325 deletions(-) create mode 100644 sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py create mode 100644 sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py create mode 100644 sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py create mode 100644 sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_prompty_strategy.py create mode 100644 sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py create mode 100644 sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_strategy.py diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index 6749ed2c5b91..e0f44299e9aa 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -4,6 +4,8 @@ ### Features Added +- Added shared model runners and evaluation strategies for asset evaluators using decision models in Microsoft Foundry. + ### Breaking Changes ### Bugs Fixed diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/__init__.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/__init__.py index 2b5182f09d18..aa848b355e25 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/__init__.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/__init__.py @@ -3,6 +3,7 @@ # --------------------------------------------------------- from ._base_eval import EvaluatorBase +from ._base_model_eval import ModelEvaluatorBase from ._base_prompty_eval import PromptyEvaluatorBase from ._base_rai_svc_eval import RaiServiceEvaluatorBase from ._base_multi_eval import MultiEvaluatorBase @@ -10,6 +11,7 @@ __all__ = [ "EvaluatorBase", + "ModelEvaluatorBase", "PromptyEvaluatorBase", "RaiServiceEvaluatorBase", "MultiEvaluatorBase", diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py new file mode 100644 index 000000000000..9636dd181533 --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py @@ -0,0 +1,328 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- + +import math +from typing import Any, Dict, List, Optional, TypeVar, Union + +from azure.core.credentials import TokenCredential +from azure.ai.evaluation._constants import EVALUATION_PASS_FAIL_MAPPING +from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget +from . import EvaluatorBase +from ._model_runners import ( + DecisionModelRunner, + PromptyModelRunner, + is_decision_model_config, +) + +T = TypeVar("T") + + +def not_applicable_result(result_key: str, reason: str, threshold: float) -> Dict[str, Any]: + return { + result_key: None, + f"{result_key}_score": None, + f"{result_key}_passed": None, + f"{result_key}_result": "not_applicable", + f"{result_key}_reason": f"Not applicable: {reason}", + f"{result_key}_status": "skipped", + f"{result_key}_threshold": threshold, + f"{result_key}_properties": None, + } + + +class ModelEvaluatorBase(EvaluatorBase[T]): + """Base class for evaluators whose judgement comes from a model. + + The base selects one model runner at construction from ``model_config`` and holds the helpers shared by + model-backed evaluators. A configuration tagged ``type="decision"`` selects a decision model runner, which + only evaluators that set ``_supports_decision_models`` accept; any other configuration selects the Prompty + runner and exposes its flow as ``_flow``. Interpreting a model's answer remains the evaluator's job. + + :param result_key: The key to use for the result of the evaluation. + :type result_key: str + :param prompty_file: The prompt path, required only for a Prompty runner. + :type prompty_file: Optional[str] + :param model_config: The model configuration to use for evaluation. + :type model_config: dict + :keyword is_reasoning_model: This parameter is in preview. If True, updates the config parameters in prompty + file based on reasoning models. Defaults to False. + :type is_reasoning_model: bool + """ + + _LLM_CALL_TIMEOUT = 600 + _DEFAULT_OPEN_API_VERSION = "2024-02-15-preview" + + _supports_decision_models = False + """Whether this evaluator can be configured with a decision model.""" + _binary_decision = False + """Whether decision answers become a binary score, which permits ``probability_threshold``.""" + + def __init__( + self, + *, + result_key: str, + prompty_file: Optional[str] = None, + model_config: dict, + eval_last_turn: bool = False, + threshold: int = 3, + credential: Optional[TokenCredential] = None, + _higher_is_better: bool = False, + **kwargs, + ) -> None: + self._result_key = result_key + self._is_reasoning_model = kwargs.get("is_reasoning_model", False) + self._prompty_file = prompty_file + self._threshold = threshold + self._higher_is_better = _higher_is_better + super().__init__(eval_last_turn=eval_last_turn, threshold=threshold, _higher_is_better=_higher_is_better) + + subclass_name = self.__class__.__name__ + if is_decision_model_config(model_config): + if not self._supports_decision_models: + raise EvaluationException( + message=f"{subclass_name} does not support decision models.", + internal_message=f"{subclass_name} does not support decision models.", + blame=ErrorBlame.USER_ERROR, + category=ErrorCategory.INVALID_VALUE, + target=ErrorTarget.EVALUATE, + ) + self._model_runner: Union[DecisionModelRunner, PromptyModelRunner] = DecisionModelRunner( + model_config, credential, binary=self._binary_decision, evaluator_name=subclass_name + ) + else: + if not prompty_file: + raise EvaluationException( + message="A prompty_file is required for a chat model evaluator.", + blame=ErrorBlame.USER_ERROR, + category=ErrorCategory.MISSING_FIELD, + target=ErrorTarget.EVALUATE, + ) + self._model_runner = PromptyModelRunner( + model_config=model_config, + prompty_file=prompty_file, + credential=credential, + is_reasoning_model=self._is_reasoning_model, + evaluator_name=subclass_name, + api_version=self._DEFAULT_OPEN_API_VERSION, + ) + + @property + def _prompty_model_runner(self) -> PromptyModelRunner: + runner = self._model_runner + if not isinstance(runner, PromptyModelRunner): + raise RuntimeError("This evaluator is not configured with a Prompty model.") + return runner + + @property + def _flow(self) -> Any: + runner = self._model_runner + if not isinstance(runner, PromptyModelRunner): + raise AttributeError("A decision model evaluator does not have a Prompty flow.") + return runner.flow + + @_flow.setter + def _flow(self, flow: Any) -> None: + self._prompty_model_runner.flow = flow + + @property + def _uses_decision_model(self) -> bool: + return isinstance(self._model_runner, DecisionModelRunner) + + @property + def _decision_model_runner(self) -> DecisionModelRunner: + runner = self._model_runner + if not isinstance(runner, DecisionModelRunner): + raise RuntimeError("This evaluator is not configured with a decision model.") + return runner + + def _get_binary_result(self, score: float) -> str: + """Get the binary result based on the score. + + :param score: The score to evaluate. + :type score: float + :return: The binary result. + :rtype: str + """ + if math.isnan(score): + return "unknown" + if self._higher_is_better: + if score >= self._threshold: + return EVALUATION_PASS_FAIL_MAPPING[True] + else: + return EVALUATION_PASS_FAIL_MAPPING[False] + else: + if score <= self._threshold: + return EVALUATION_PASS_FAIL_MAPPING[True] + else: + return EVALUATION_PASS_FAIL_MAPPING[False] + + @staticmethod + def _get_token_metadata(prompty_output: Dict) -> Dict: + """Extract token usage and model metadata from the prompty output dict. + + :param prompty_output: The raw output dictionary from the prompty flow. + :type prompty_output: Dict + :return: A dictionary with token counts, finish reason, model, and sample I/O. + :rtype: Dict + """ + return { + "prompt_tokens": prompty_output.get("input_token_count", 0), + "completion_tokens": prompty_output.get("output_token_count", 0), + "total_tokens": prompty_output.get("total_token_count", 0), + "finish_reason": prompty_output.get("finish_reason", ""), + "model": prompty_output.get("model_id", ""), + "sample_input": prompty_output.get("sample_input", ""), + "sample_output": prompty_output.get("sample_output", ""), + } + + @staticmethod + def _get_built_in_tool_definition(tool_name: str): + """Get the definition for the built-in tool.""" + try: + from ..._converters._models import _BUILT_IN_DESCRIPTIONS, _BUILT_IN_PARAMS + + if tool_name in _BUILT_IN_DESCRIPTIONS: + return { + "type": tool_name, + "description": _BUILT_IN_DESCRIPTIONS[tool_name], + "name": tool_name, + "parameters": _BUILT_IN_PARAMS.get(tool_name, {}), + } + except ImportError: + pass + return None + + def _get_needed_built_in_tool_definitions(self, tool_calls: List[Dict]) -> List[Dict]: + """Extract tool definitions needed for the given built-in tool calls.""" + needed_definitions = [] + for tool_call in tool_calls: + if isinstance(tool_call, dict): + tool_type = tool_call.get("type") + + # Only support converter format: {type: "tool_call", name: "bing_custom_search", arguments: {...}} + if tool_type == "tool_call": + tool_name = tool_call.get("name") + if tool_name: + definition = self._get_built_in_tool_definition(tool_name) + if definition and definition not in needed_definitions: + needed_definitions.append(definition) + + return needed_definitions + + def _extract_tool_names_from_calls(self, tool_calls: List[Dict]) -> List[str]: + """Extract just the tool names from tool calls, removing parameters.""" + tool_names = [] + for tool_call in tool_calls: + if isinstance(tool_call, dict): + tool_type = tool_call.get("type") + if tool_type == "tool_call": + tool_name = tool_call.get("name") + if tool_name: + tool_names.append(tool_name) + elif tool_call.get("function", {}).get("name"): + # Handle function call format + tool_names.append(tool_call["function"]["name"]) + elif tool_call.get("name"): + # Handle direct name format + tool_names.append(tool_call["name"]) + return tool_names + + def _extract_needed_tool_definitions( + self, tool_calls: List[Dict], tool_definitions: List[Dict], error_target: ErrorTarget + ) -> List[Dict]: + """Extract the tool definitions that are needed for the provided tool calls. + + :param tool_calls: The tool calls that need definitions + :type tool_calls: List[Dict] + :param tool_definitions: User-provided tool definitions + :type tool_definitions: List[Dict] + :param error_target: The evaluator-specific error target for exceptions + :type error_target: ErrorTarget + :return: List of needed tool definitions + :rtype: List[Dict] + :raises EvaluationException: If validation fails + """ + needed_tool_definitions = [] + + # Add all user-provided tool definitions + needed_tool_definitions.extend(tool_definitions) + + # Add the needed built-in tool definitions (if they are called) + built_in_definitions = self._get_needed_built_in_tool_definitions(tool_calls) + needed_tool_definitions.extend(built_in_definitions) + + # An OpenAPI tool is a collection of functions. When an OpenAPI tool + # definition exposes its nested functions, expand it into those functions + # so tool calls referencing a nested function name can be matched. If an + # OpenAPI tool definition has no nested functions, keep the OpenAPI tool + # definition as is for backward compatibility. + tool_definitions_expanded = [] + for tool in needed_tool_definitions: + openapi_functions = tool.get("functions") if tool.get("type") == "openapi" else None + if openapi_functions: + tool_definitions_expanded.extend(openapi_functions) + else: + tool_definitions_expanded.append(tool) + + # Validate that all tool calls have corresponding definitions + for tool_call in tool_calls: + if isinstance(tool_call, dict): + tool_type = tool_call.get("type") + + if tool_type == "tool_call": + tool_name = tool_call.get("name") + if tool_name and self._get_built_in_tool_definition(tool_name): + # This is a built-in tool from converter, already handled above + continue + elif tool_name: + # This is a regular function tool from converter + tool_definition_exists = any( + tool.get("name") == tool_name for tool in tool_definitions_expanded + ) + if not tool_definition_exists: + raise EvaluationException( + message=f"Tool definition for {tool_name} not found", + blame=ErrorBlame.USER_ERROR, + category=ErrorCategory.INVALID_VALUE, + target=error_target, + ) + else: + raise EvaluationException( + message=f"Tool call missing name: {tool_call}", + blame=ErrorBlame.USER_ERROR, + category=ErrorCategory.INVALID_VALUE, + target=error_target, + ) + else: + # Unsupported tool format - only converter format is supported + raise EvaluationException( + message=f"Unsupported tool call format. Only converter format is supported: {tool_call}", + blame=ErrorBlame.USER_ERROR, + category=ErrorCategory.INVALID_VALUE, + target=error_target, + ) + else: + # Tool call is not a dictionary + raise EvaluationException( + message=f"Tool call is not a dictionary: {tool_call}", + blame=ErrorBlame.USER_ERROR, + category=ErrorCategory.INVALID_VALUE, + target=error_target, + ) + + return needed_tool_definitions + + def _return_not_applicable_result( + self, error_message: str, threshold: Union[int, float] + ) -> Dict[str, Union[str, float, Dict, None]]: + """Return a result indicating that the tool call is not applicable for evaluation. + + :param error_message: The error message indicating why the evaluation is not applicable. + :type error_message: str + :param threshold: The threshold value for the evaluation. + :type threshold: Union[int, float] + :return: A dictionary containing the result of the evaluation. + :rtype: Dict[str, Union[str, float, None]] + """ + return not_applicable_result(self._result_key, error_message, threshold) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_prompty_eval.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_prompty_eval.py index b2c5f6bd81d0..253d9ca5154e 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_prompty_eval.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_prompty_eval.py @@ -2,50 +2,31 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # --------------------------------------------------------- -import json import math -import re -import os -from typing import Dict, Optional, TypeVar, Union, List +from typing import Dict, TypeVar, Union -if os.getenv("AI_EVALS_USE_PF_PROMPTY", "false").lower() == "true": - from promptflow.core._flow import AsyncPrompty -else: - from azure.ai.evaluation._legacy.prompty import AsyncPrompty from typing_extensions import override -from azure.core.credentials import TokenCredential -from azure.ai.evaluation._common.constants import PROMPT_BASED_REASON_EVALUATORS -from azure.ai.evaluation._constants import EVALUATION_PASS_FAIL_MAPPING from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget -from ..._common.utils import construct_prompty_model_config, validate_model_config, parse_quality_evaluator_reason_score # The message-preprocessing helpers now live in ``azure.ai.evaluation._common.utils`` (single source of # truth). They are imported here so this module can use them and so callers that historically imported # them from this module keep working. ``_drop_mcp_approval_messages`` / ``_normalize_function_call_types`` # are re-exported only (used indirectly via ``_preprocess_messages``). -from ..._common.utils import ( # pylint: disable=unused-import +from ..._common.utils import ( # noqa: F401 # pylint: disable=unused-import _is_intermediate_response, _drop_mcp_approval_messages, _normalize_function_call_types, _preprocess_messages, ) -from . import EvaluatorBase - -try: - from ..._user_agent import UserAgentSingleton -except ImportError: - - class UserAgentSingleton: - @property - def value(self) -> str: - return "None" - +from ._base_model_eval import ModelEvaluatorBase +from ._prompty_strategy import PromptyEvaluationStrategy +from ._model_runners import AsyncPrompty, UserAgentSingleton # noqa: F401 # pylint: disable=unused-import T = TypeVar("T") -class PromptyEvaluatorBase(EvaluatorBase[T]): +class PromptyEvaluatorBase(ModelEvaluatorBase[T]): """Base class for all evaluators that make use of context as an input. It's also assumed that such evaluators make use of a prompty file, and return their results as a dictionary, with a single key-value pair linking the result name to a float value (unless multi-turn evaluation occurs, in which case the @@ -61,69 +42,13 @@ class PromptyEvaluatorBase(EvaluatorBase[T]): :param ignore_queries: If True, queries will be ignored in conversation evaluations. Default is False. Useful since some evaluators of this format are response-only. :type ignore_queries: bool - :keyword is_reasoning_model: This parameter is in preview. If True, updates the config parameters in prompty file based on reasoning models. Defaults to False. + :keyword is_reasoning_model: This parameter is in preview. If True, updates the config parameters in + prompty file based on reasoning models. Defaults to False. :type is_reasoning_model: bool """ - _LLM_CALL_TIMEOUT = 600 - _DEFAULT_OPEN_API_VERSION = "2024-02-15-preview" - - def __init__( - self, - *, - result_key: str, - prompty_file: str, - model_config: dict, - eval_last_turn: bool = False, - threshold: int = 3, - credential: Optional[TokenCredential] = None, - _higher_is_better: bool = False, - **kwargs, - ) -> None: - self._result_key = result_key - self._is_reasoning_model = kwargs.get("is_reasoning_model", False) - self._prompty_file = prompty_file - self._threshold = threshold - self._higher_is_better = _higher_is_better - super().__init__(eval_last_turn=eval_last_turn, threshold=threshold, _higher_is_better=_higher_is_better) - - subclass_name = self.__class__.__name__ - user_agent = f"{UserAgentSingleton().value} (type=evaluator subtype={subclass_name})" - prompty_model_config = construct_prompty_model_config( - validate_model_config(model_config), - self._DEFAULT_OPEN_API_VERSION, - user_agent, - ) - - self._flow = AsyncPrompty.load( - source=self._prompty_file, - model=prompty_model_config, - token_credential=credential, - is_reasoning_model=self._is_reasoning_model, - ) - - # __call__ not overridden here because child classes have such varied signatures that there's no point - # defining a default here. - def _get_binary_result(self, score: float) -> str: - """Get the binary result based on the score. - - :param score: The score to evaluate. - :type score: float - :return: The binary result. - :rtype: str - """ - if math.isnan(score): - return "unknown" - if self._higher_is_better: - if score >= self._threshold: - return EVALUATION_PASS_FAIL_MAPPING[True] - else: - return EVALUATION_PASS_FAIL_MAPPING[False] - else: - if score <= self._threshold: - return EVALUATION_PASS_FAIL_MAPPING[True] - else: - return EVALUATION_PASS_FAIL_MAPPING[False] + # __init__ and __call__ are inherited: the constructor from ModelEvaluatorBase, and __call__ is not defined + # because child classes have such varied signatures that there's no point defining a default here. @override async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]: # type: ignore[override] @@ -158,243 +83,12 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]: # t if isinstance(eval_input.get("query"), list): eval_input["query"] = _preprocess_messages(eval_input["query"]) - # Call the prompty flow to get the evaluation result. - prompty_output_dict = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input) - - score = math.nan - reason = "" - llm_properties = {} - - if prompty_output_dict: - llm_output = prompty_output_dict.get("llm_output", "") - - # Parse JSON output from LLM - parsed_output = None - if isinstance(llm_output, dict): - parsed_output = llm_output - elif isinstance(llm_output, str): - try: - parsed_output = json.loads(llm_output) - except (json.JSONDecodeError, TypeError): - parsed_output = None - - if parsed_output and isinstance(parsed_output, dict): - # Handle skipped status from LLM - llm_status = parsed_output.get("status", "completed") - if llm_status == "skipped": - skip_reason = parsed_output.get("reason", "") - return self._return_not_applicable_result(skip_reason, self._threshold) - - score = parsed_output.get("score", math.nan) - reason = parsed_output.get("reason", "") - llm_properties = parsed_output.get("properties", {}) or {} - else: - # Fallback: try to parse legacy XML format or extract digit - if isinstance(llm_output, str) and self._result_key in PROMPT_BASED_REASON_EVALUATORS: - score, reason = parse_quality_evaluator_reason_score(llm_output) - elif isinstance(llm_output, str): - match = re.search(r"\d", llm_output) - if match: - score = float(match.group()) - - score = float(score) if score is not None else math.nan - score_result = self._get_binary_result(score) - - llm_properties.update(self._get_token_metadata(prompty_output_dict)) - - return { - self._result_key: score, - f"{self._result_key}_score": score, - f"{self._result_key}_passed": score_result == "pass", - f"{self._result_key}_result": score_result, - f"{self._result_key}_reason": reason, - f"{self._result_key}_status": "completed", - f"{self._result_key}_threshold": self._threshold, - f"{self._result_key}_properties": llm_properties, - } - - raise EvaluationException( - message="Evaluator returned invalid output.", - blame=ErrorBlame.SYSTEM_ERROR, - category=ErrorCategory.FAILED_EXECUTION, - target=ErrorTarget.EVALUATE, - ) - - @staticmethod - def _get_token_metadata(prompty_output: Dict) -> Dict: - """Extract token usage and model metadata from the prompty output dict. - - :param prompty_output: The raw output dictionary from the prompty flow. - :type prompty_output: Dict - :return: A dictionary with token counts, finish reason, model, and sample I/O. - :rtype: Dict - """ - return { - "prompt_tokens": prompty_output.get("input_token_count", 0), - "completion_tokens": prompty_output.get("output_token_count", 0), - "total_tokens": prompty_output.get("total_token_count", 0), - "finish_reason": prompty_output.get("finish_reason", ""), - "model": prompty_output.get("model_id", ""), - "sample_input": prompty_output.get("sample_input", ""), - "sample_output": prompty_output.get("sample_output", ""), - } - - @staticmethod - def _get_built_in_tool_definition(tool_name: str): - """Get the definition for the built-in tool.""" - try: - from ..._converters._models import _BUILT_IN_DESCRIPTIONS, _BUILT_IN_PARAMS - - if tool_name in _BUILT_IN_DESCRIPTIONS: - return { - "type": tool_name, - "description": _BUILT_IN_DESCRIPTIONS[tool_name], - "name": tool_name, - "parameters": _BUILT_IN_PARAMS.get(tool_name, {}), - } - except ImportError: - pass - return None - - def _get_needed_built_in_tool_definitions(self, tool_calls: List[Dict]) -> List[Dict]: - """Extract tool definitions needed for the given built-in tool calls.""" - needed_definitions = [] - for tool_call in tool_calls: - if isinstance(tool_call, dict): - tool_type = tool_call.get("type") - - # Only support converter format: {type: "tool_call", name: "bing_custom_search", arguments: {...}} - if tool_type == "tool_call": - tool_name = tool_call.get("name") - if tool_name: - definition = self._get_built_in_tool_definition(tool_name) - if definition and definition not in needed_definitions: - needed_definitions.append(definition) - - return needed_definitions - - def _extract_tool_names_from_calls(self, tool_calls: List[Dict]) -> List[str]: - """Extract just the tool names from tool calls, removing parameters.""" - tool_names = [] - for tool_call in tool_calls: - if isinstance(tool_call, dict): - tool_type = tool_call.get("type") - if tool_type == "tool_call": - tool_name = tool_call.get("name") - if tool_name: - tool_names.append(tool_name) - elif tool_call.get("function", {}).get("name"): - # Handle function call format - tool_names.append(tool_call["function"]["name"]) - elif tool_call.get("name"): - # Handle direct name format - tool_names.append(tool_call["name"]) - return tool_names - - def _extract_needed_tool_definitions( - self, tool_calls: List[Dict], tool_definitions: List[Dict], error_target: ErrorTarget - ) -> List[Dict]: - """Extract the tool definitions that are needed for the provided tool calls. - - :param tool_calls: The tool calls that need definitions - :type tool_calls: List[Dict] - :param tool_definitions: User-provided tool definitions - :type tool_definitions: List[Dict] - :param error_target: The evaluator-specific error target for exceptions - :type error_target: ErrorTarget - :return: List of needed tool definitions - :rtype: List[Dict] - :raises EvaluationException: If validation fails - """ - needed_tool_definitions = [] - - # Add all user-provided tool definitions - needed_tool_definitions.extend(tool_definitions) - - # Add the needed built-in tool definitions (if they are called) - built_in_definitions = self._get_needed_built_in_tool_definitions(tool_calls) - needed_tool_definitions.extend(built_in_definitions) - - # An OpenAPI tool is a collection of functions. When an OpenAPI tool - # definition exposes its nested functions, expand it into those functions - # so tool calls referencing a nested function name can be matched. If an - # OpenAPI tool definition has no nested functions, keep the OpenAPI tool - # definition as is for backward compatibility. - tool_definitions_expanded = [] - for tool in needed_tool_definitions: - openapi_functions = tool.get("functions") if tool.get("type") == "openapi" else None - if openapi_functions: - tool_definitions_expanded.extend(openapi_functions) - else: - tool_definitions_expanded.append(tool) - - # Validate that all tool calls have corresponding definitions - for tool_call in tool_calls: - if isinstance(tool_call, dict): - tool_type = tool_call.get("type") - - if tool_type == "tool_call": - tool_name = tool_call.get("name") - if tool_name and self._get_built_in_tool_definition(tool_name): - # This is a built-in tool from converter, already handled above - continue - elif tool_name: - # This is a regular function tool from converter - tool_definition_exists = any( - tool.get("name") == tool_name for tool in tool_definitions_expanded - ) - if not tool_definition_exists: - raise EvaluationException( - message=f"Tool definition for {tool_name} not found", - blame=ErrorBlame.USER_ERROR, - category=ErrorCategory.INVALID_VALUE, - target=error_target, - ) - else: - raise EvaluationException( - message=f"Tool call missing name: {tool_call}", - blame=ErrorBlame.USER_ERROR, - category=ErrorCategory.INVALID_VALUE, - target=error_target, - ) - else: - # Unsupported tool format - only converter format is supported - raise EvaluationException( - message=f"Unsupported tool call format. Only converter format is supported: {tool_call}", - blame=ErrorBlame.USER_ERROR, - category=ErrorCategory.INVALID_VALUE, - target=error_target, - ) - else: - # Tool call is not a dictionary - raise EvaluationException( - message=f"Tool call is not a dictionary: {tool_call}", - blame=ErrorBlame.USER_ERROR, - category=ErrorCategory.INVALID_VALUE, - target=error_target, - ) - - return needed_tool_definitions - - def _return_not_applicable_result( - self, error_message: str, threshold: Union[int, float] - ) -> Dict[str, Union[str, float, Dict, None]]: - """Return a result indicating that the tool call is not applicable for evaluation. - - :param error_message: The error message indicating why the evaluation is not applicable. - :type error_message: str - :param threshold: The threshold value for the evaluation. - :type threshold: Union[int, float] - :return: A dictionary containing the result of the evaluation. - :rtype: Dict[str, Union[str, float, None]] - """ - return { - f"{self._result_key}": None, - f"{self._result_key}_score": None, - f"{self._result_key}_passed": None, - f"{self._result_key}_result": "not_applicable", - f"{self._result_key}_reason": f"Not applicable: {error_message}", - f"{self._result_key}_status": "skipped", - f"{self._result_key}_threshold": threshold, - f"{self._result_key}_properties": None, - } + return await PromptyEvaluationStrategy( + self._prompty_model_runner, + result_key=self._result_key, + threshold=self._threshold, + timeout=self._LLM_CALL_TIMEOUT, + strict=False, + missing_score=math.nan, + higher_is_better=self._higher_is_better, + ).evaluate(eval_input) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py new file mode 100644 index 000000000000..3f4c171565c8 --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py @@ -0,0 +1,189 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- + +"""Evaluates prepared input with a decision model and returns the final evaluator result. + +An evaluator supplies its questions, scoring policy and error target; this strategy owns the shared mechanics: +status gating, score mapping, provenance and result shape. +""" + +import hashlib +import json +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Tuple, TypeVar, Union + +from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget +from ._base_model_eval import ModelEvaluatorBase, not_applicable_result +from ._model_runners import DecisionModelRunner, validate_noul_answer, validate_score_answer + +STATUS_CUTOFF = 0.5 +T = TypeVar("T") + + +def _checked_value(validate: Callable[..., T], error_target: ErrorTarget, *args: Any) -> T: + try: + return validate(*args) + except ValueError as exc: + message = f"The decision model returned an invalid response: {exc}" + raise EvaluationException( + message=message, + internal_message=message, + blame=ErrorBlame.SYSTEM_ERROR, + category=ErrorCategory.FAILED_EXECUTION, + target=error_target, + ) from None + + +def decision_probability( + response: Dict[str, Any], question_id: str, error_target: ErrorTarget = ErrorTarget.EVALUATE +) -> float: + """Read and validate the probability of a binary decision answer.""" + return _checked_value(validate_noul_answer, error_target, response["answers"][question_id]) + + +def decision_distribution( + response: Dict[str, Any], question_id: str, levels: int, error_target: ErrorTarget = ErrorTarget.EVALUATE +) -> List[float]: + """Read and validate the distribution of a scored decision answer.""" + return _checked_value(validate_score_answer, error_target, response["answers"][question_id], levels) + + +def decision_skip_reason( + response: Dict[str, Any], cutoff: float, error_target: ErrorTarget = ErrorTarget.EVALUATE +) -> Optional[str]: + """Apply the evaluator's status cutoff before interpreting a score.""" + probability = decision_probability(response, "status", error_target) + if probability >= cutoff: + return None + return ( + f"Status completion probability {probability:.4f} is below {cutoff:.4f}; " + "evaluation skipped. No model rationale is available." + ) + + +def decision_token_counts(response: Dict[str, Any]) -> Dict[str, Any]: + """Convert decision usage to the common model-output metadata shape.""" + usage = response["usage"] + return { + "input_token_count": usage["input_tokens"], + "output_token_count": usage["output_tokens"], + "total_token_count": usage["input_tokens"] + usage["output_tokens"], + "model_id": response["model"], + } + + +def _label(result_key: str) -> str: + return result_key.replace("_", " ") + + +def _digest(question: Dict[str, Any]) -> str: + return hashlib.sha256(json.dumps(question, sort_keys=True).encode("utf-8")).hexdigest() + + +@dataclass(frozen=True) +class BinaryScoring: + """Scores 1 when the answer's probability reaches the cutoff, otherwise 0.""" + + probability_cutoff: float + + def interpret( + self, response: Dict[str, Any], result_key: str, error_target: ErrorTarget + ) -> Tuple[float, str, Dict[str, Any]]: + probability = decision_probability(response, result_key, error_target) + score = float(probability >= self.probability_cutoff) + relation = "at or above" if score else "below" + reason = ( + f"{_label(result_key).capitalize()} probability {probability:.4f} is {relation} " + f"the {self.probability_cutoff:.4f} cutoff." + ) + details = { + "probability_threshold": self.probability_cutoff, + "probabilities": {"false": 1 - probability, "true": probability}, + } + return score, reason, details + + def passes(self, score: float, threshold: float) -> bool: + return score >= 1.0 + + +@dataclass(frozen=True) +class LikertScoring: + """Scores the highest-probability level of a scale starting at ``minimum``; ties select the lowest level.""" + + levels: int + minimum: int = 1 + + def interpret( + self, response: Dict[str, Any], result_key: str, error_target: ErrorTarget + ) -> Tuple[float, str, Dict[str, Any]]: + distribution = decision_distribution(response, result_key, self.levels, error_target) + score = float(max(range(self.levels), key=distribution.__getitem__) + self.minimum) + answer = response["answers"][result_key] + reason = f"Selected {_label(result_key)} level {score:g} from the highest-probability level." + details = { + "probabilities": answer["probabilities"], + "confidence": answer["confidence"], + "native_mean": answer["score"], + "expected_score": sum(level * value for level, value in enumerate(distribution)) + self.minimum, + } + return score, reason, details + + def passes(self, score: float, threshold: float) -> bool: + return score >= threshold + + +class DecisionEvaluationStrategy: + def __init__( + self, + runner: DecisionModelRunner, + *, + result_key: str, + error_target: ErrorTarget, + questions: Dict[str, Any], + scoring: Union[BinaryScoring, LikertScoring], + threshold: float, + timeout: float, + extra_properties: Optional[Callable[[Dict[str, Any]], Dict[str, Any]]] = None, + ): + self.runner = runner + self.result_key = result_key + self.error_target = error_target + self.questions = questions + self.scoring = scoring + self.threshold = threshold + self.timeout = timeout + self.extra_properties = extra_properties + + async def evaluate(self, eval_input: Dict[str, Any]) -> Dict[str, Any]: + key = self.result_key + response = await self.runner(state=dict(eval_input), questions=self.questions, timeout=self.timeout) + skip_reason = decision_skip_reason(response, STATUS_CUTOFF, self.error_target) + if skip_reason: + return not_applicable_result(key, skip_reason, self.threshold) + score, reason, details = self.scoring.interpret(response, key, self.error_target) + properties = self.extra_properties(eval_input) if self.extra_properties else {} + properties["decision_model"] = { + "deployment": self.runner.deployment, + "model": response["model"], + "rubric_hash": _digest(self.questions[key]), + "status_rubric_hash": _digest(self.questions["status"]), + "native_status_answer": response["answers"]["status"], + "status_completion_probability": decision_probability(response, "status", self.error_target), + "status_probability_threshold": STATUS_CUTOFF, + "reason_type": "templated", + **details, + "native_answer": response["answers"][key], + } + properties.update(ModelEvaluatorBase._get_token_metadata(decision_token_counts(response))) + passed = self.scoring.passes(score, self.threshold) + return { + key: score, + f"{key}_score": score, + f"{key}_passed": passed, + f"{key}_result": "pass" if passed else "fail", + f"{key}_reason": f"{reason} No model rationale is available.", + f"{key}_status": "completed", + f"{key}_threshold": self.threshold, + f"{key}_properties": properties, + } diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py new file mode 100644 index 000000000000..193ebdf3fa99 --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py @@ -0,0 +1,318 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- + +"""Model runners execute a prepared request against a model backend. + +A runner owns backend configuration, authentication and transport. It never interprets a model's answer +as an evaluator score; that stays with the evaluator. +""" + +import asyncio +import inspect +import json +import math +import os +from typing import Any, Dict, List + +from azure.core.pipeline.policies import AsyncRedirectPolicy, AsyncRetryPolicy + +from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException +from ..._common.utils import construct_prompty_model_config, validate_model_config +from ..._http_utils import get_async_http_client + +if os.getenv("AI_EVALS_USE_PF_PROMPTY", "false").lower() == "true": + from promptflow.core._flow import AsyncPrompty +else: + from azure.ai.evaluation._legacy.prompty import AsyncPrompty + +try: + from ..._user_agent import UserAgentSingleton +except ImportError: + + class UserAgentSingleton: # type: ignore[no-redef] + @property + def value(self) -> str: + return "None" + + +DECISION_MODEL_TYPE = "decision" +_DECISION_PATH = "/providers/microsoft/v1/systemone" +_DECISION_TOKEN_SCOPE = "https://cognitiveservices.azure.com/.default" +_DECISION_RETRY_STATUS_CODES = [408, 429, 500, 502, 503, 504, 529] +_DECISION_CONFIG_KEYS = { + "type", + "azure_endpoint", + "azure_deployment", + "api_key", + "credential", + "probability_threshold", +} +_DEFAULT_PROBABILITY_THRESHOLD = 0.75 +# The service adds api_version to every resolved model configuration; the decision endpoint does not use it. +_IGNORED_CONFIG_KEYS = {"api_version"} + + +def is_decision_model_config(model_config: Any) -> bool: + """Whether a model configuration explicitly selects a decision model. + + :param model_config: The model configuration passed to an evaluator. + :type model_config: Any + :return: True only for configurations tagged ``type="decision"``. + :rtype: bool + """ + return isinstance(model_config, dict) and model_config.get("type") == DECISION_MODEL_TYPE + + +class PromptyModelRunner: + """Loads a Prompty flow for a chat model and invokes it without interpreting the output.""" + + def __init__( + self, + *, + model_config: dict, + prompty_file: str, + credential: Any = None, + is_reasoning_model: bool = False, + evaluator_name: str, + api_version: str, + ) -> None: + user_agent = f"{UserAgentSingleton().value} (type=evaluator subtype={evaluator_name})" + prompty_model_config = construct_prompty_model_config( + validate_model_config(model_config), api_version, user_agent + ) + self.flow = AsyncPrompty.load( + source=prompty_file, + model=prompty_model_config, + token_credential=credential, + is_reasoning_model=is_reasoning_model, + ) + + async def __call__(self, *, timeout: float, **eval_input: Any) -> Dict[str, Any]: + return await self.flow(timeout=timeout, **eval_input) + + +def _invalid_config(message: str) -> EvaluationException: + return EvaluationException( + message=message, + internal_message=message, + blame=ErrorBlame.USER_ERROR, + category=ErrorCategory.INVALID_VALUE, + target=ErrorTarget.EVALUATE, + ) + + +def _failed(message: str, blame: ErrorBlame = ErrorBlame.SYSTEM_ERROR) -> EvaluationException: + return EvaluationException( + message=message, + internal_message=message, + blame=blame, + category=ErrorCategory.FAILED_EXECUTION, + target=ErrorTarget.EVALUATE, + ) + + +def _probability(value: Any) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value): + raise ValueError("Decision probabilities must be finite numbers.") + if not 0 <= value <= 1: + raise ValueError("Decision probabilities must be in [0, 1].") + return float(value) + + +def validate_noul_answer(answer: Dict[str, Any]) -> float: + """Return the probability of a ``noul`` answer. + + :param answer: The answer for a ``noul`` question. + :type answer: Dict[str, Any] + :return: The probability that the condition holds. + :rtype: float + :raises ValueError: If the probability is missing, non-finite or outside [0, 1]. + """ + return _probability(answer.get("noul")) + + +def validate_score_answer(answer: Dict[str, Any], levels: int) -> List[float]: + """Return the level probabilities of a ``score`` answer after validating its fields. + + :param answer: The answer for a ``score`` question. + :type answer: Dict[str, Any] + :param levels: The number of levels in the question's scale. + :type levels: int + :return: The probability of each level, lowest level first. + :rtype: List[float] + :raises ValueError: If the probabilities, mean or confidence are missing or malformed. + """ + probabilities = answer.get("probabilities") + if not isinstance(probabilities, dict) or set(probabilities) != {str(i) for i in range(levels)}: + raise ValueError(f"a score answer requires probabilities for exactly {levels} levels.") + distribution = [_probability(probabilities[str(i)]) for i in range(levels)] + mean = answer.get("score") + if isinstance(mean, bool) or not isinstance(mean, (int, float)) or not math.isfinite(mean): + raise ValueError("the score mean must be finite.") + _probability(answer.get("confidence")) + return distribution + + +class DecisionModelRunner: + """Sends prepared evidence and typed questions to a Microsoft-Decision-1 deployment in Foundry. + + The model configuration must be tagged ``type="decision"`` and contain the Foundry resource endpoint + (``azure_endpoint``) and the deployment name (``azure_deployment``). Authentication uses ``api_key`` when + present, otherwise an Entra credential (from ``credential`` in the configuration or the constructor, or the + SDK default). ``api_version`` is accepted and ignored. + ``probability_threshold`` (default 0.75) is only accepted when ``binary=True``. + """ + + def __init__( + self, + model_config: dict, + credential: Any = None, + *, + binary: bool = False, + evaluator_name: str = "DecisionModelRunner", + ) -> None: + if not is_decision_model_config(model_config): + raise _invalid_config(f"A decision model configuration must be a dict with type '{DECISION_MODEL_TYPE}'.") + unexpected = set(model_config) - _DECISION_CONFIG_KEYS - _IGNORED_CONFIG_KEYS + if unexpected: + raise _invalid_config(f"Unsupported decision model configuration fields: {sorted(unexpected)}.") + + endpoint = model_config.get("azure_endpoint") + if not isinstance(endpoint, str) or not endpoint.strip(): + raise _invalid_config("'azure_endpoint' must be a nonempty string.") + deployment = model_config.get("azure_deployment") + if not isinstance(deployment, str) or not deployment.strip(): + raise _invalid_config("'azure_deployment' must be the decision model deployment name.") + api_key = model_config.get("api_key") + if api_key is not None and (not isinstance(api_key, str) or not api_key.strip()): + raise _invalid_config("'api_key' must be a nonempty string.") + + # Like the Prompty path, an API key wins; a credential is only used when there is no key. + credential = credential if credential is not None else model_config.get("credential") + self._credential = None if api_key else credential + + threshold = model_config.get("probability_threshold", _DEFAULT_PROBABILITY_THRESHOLD) + if "probability_threshold" in model_config and not binary: + raise _invalid_config("'probability_threshold' is only supported by binary evaluators.") + try: + self.probability_threshold = _probability(threshold) + except ValueError: + raise _invalid_config("'probability_threshold' must be a number in [0, 1].") from None + + self._url = f"{endpoint.rstrip('/')}{_DECISION_PATH}" + self._deployment = deployment + self._api_key = api_key + self._user_agent = f"{UserAgentSingleton().value} (type=evaluator subtype={evaluator_name})" + + @property + def deployment(self) -> str: + """The deployment name sent as the request's model.""" + return self._deployment + + async def __call__( + self, *, state: Any, questions: Dict[str, Dict[str, Any]], timeout: float + ) -> Dict[str, Any]: + """Request the decisions for ``questions`` about ``state`` and return the validated response. + + :keyword state: The prepared evidence, as text or JSON-serializable data. + :paramtype state: Any + :keyword questions: Typed questions keyed by identifier; ``noul`` and ``score`` are supported. + :paramtype questions: Dict[str, Dict[str, Any]] + :keyword timeout: Overall time limit in seconds. + :paramtype timeout: float + :return: ``{"model", "answers", "usage"}`` with the response's structure checked against the questions. + :rtype: Dict[str, Any] + """ + for question in questions.values(): + if question.get("type") not in ("noul", "score"): + raise _invalid_config(f"Decision question type '{question.get('type')}' is not supported.") + try: + payload = await asyncio.wait_for(self._post(state, questions), timeout=timeout) + except asyncio.TimeoutError: + raise _failed("The decision model request timed out.") from None + try: + return self._validate_response(payload, questions) + except ValueError as exc: + raise _failed(f"The decision model returned an invalid response: {exc}") from None + + async def _get_token(self) -> str: + if self._credential is None: + return await asyncio.to_thread(asyncio.run, self._get_default_token()) + token = await asyncio.to_thread(self._credential.get_token, _DECISION_TOKEN_SCOPE) + if inspect.isawaitable(token): + token = await token + return token.token + + @staticmethod + async def _get_default_token() -> str: + from azure.ai.evaluation._legacy._common._async_token_provider import AsyncAzureTokenProvider + + async with AsyncAzureTokenProvider() as provider: + return (await provider.get_token(_DECISION_TOKEN_SCOPE)).token + + async def _post(self, state: Any, questions: Dict[str, Dict[str, Any]]) -> Any: + headers = {"x-ms-useragent": self._user_agent} + if self._api_key: + headers["api-key"] = self._api_key + else: + try: + headers["Authorization"] = f"Bearer {await self._get_token()}" + except EvaluationException: + raise + except Exception: # pylint: disable=broad-except + raise _failed("Could not acquire a token for the decision model.", ErrorBlame.USER_ERROR) from None + + body = {"model": self._deployment, "state": state, "questions": questions} + policies = { + "retry_policy": AsyncRetryPolicy( + retry_total=3, + retry_backoff_factor=0.8, + retry_on_status_codes=_DECISION_RETRY_STATUS_CODES, + ), + "redirect_policy": AsyncRedirectPolicy(permit_redirects=False), + } + try: + async with get_async_http_client().with_policies(**policies) as client: + response = await client.post(self._url, json=body, headers=headers, retry_on_methods=["POST"]) + status = response.status_code + content = response.body() if status == 200 else b"" + except Exception: # pylint: disable=broad-except + raise _failed("The decision model request failed; check the endpoint and network access.") from None + if status != 200: + blame = ErrorBlame.USER_ERROR if status in (400, 401, 403, 404, 422) else ErrorBlame.SYSTEM_ERROR + raise _failed(f"The decision model returned HTTP {status}.", blame) + try: + return json.loads(content) + except (ValueError, TypeError): + raise _failed("The decision model returned invalid JSON.") from None + + @staticmethod + def _validate_response(payload: Any, questions: Dict[str, Dict[str, Any]]) -> Dict[str, Any]: + # Only the response's structure is checked here; answer values are validated by the evaluator after it + # has read the status answer, so a skipped evaluation never depends on an unusable score. + if not isinstance(payload, dict) or not isinstance(payload.get("answers"), dict): + raise ValueError("answers are missing.") + answers = payload["answers"] + if set(answers) != set(questions): + raise ValueError("answer identifiers do not match the questions.") + for question_id, question in questions.items(): + answer = answers[question_id] + question_type = question.get("type") + if not isinstance(answer, dict) or answer.get("type") != question_type: + raise ValueError(f"answer '{question_id}' does not match its question type.") + + usage = payload.get("usage") + if not isinstance(usage, dict): + raise ValueError("usage is missing.") + for name in ("input_tokens", "output_tokens"): + if isinstance(usage.get(name), bool) or not isinstance(usage.get(name), int) or usage[name] < 0: + raise ValueError("usage must contain nonnegative integer token counts.") + model = payload.get("model") + if not isinstance(model, str) or not model.strip(): + raise ValueError("the model identifier is missing.") + return { + "model": model, + "answers": answers, + "usage": {"input_tokens": usage["input_tokens"], "output_tokens": usage["output_tokens"]}, + } diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_prompty_strategy.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_prompty_strategy.py new file mode 100644 index 000000000000..93ad2eed104e --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_prompty_strategy.py @@ -0,0 +1,135 @@ +import json +import math +import re +from typing import Any, Callable, Dict, Optional, Tuple + +from azure.ai.evaluation._common.constants import PROMPT_BASED_REASON_EVALUATORS +from azure.ai.evaluation._common.utils import check_score_is_valid, parse_quality_evaluator_reason_score +from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget +from ._base_model_eval import ModelEvaluatorBase, not_applicable_result + + +class PromptyEvaluationStrategy: + def __init__( + self, + runner: Callable, + *, + result_key: str, + threshold: float, + timeout: float, + strict: bool = True, + missing_score: Optional[float] = 0, + score_range: Optional[Tuple[int, int]] = None, + error_target: Any = ErrorTarget.EVALUATE, + score_error_target: Any = None, + binary_cutoff: Optional[float] = None, + higher_is_better: bool = True, + completed_alias: Optional[str] = None, + top_level_usage: bool = False, + flow_resolver: Optional[Callable] = None, + ): + self.runner = runner + self.result_key = result_key + self.threshold = threshold + self.timeout = timeout + self.strict = strict + self.missing_score = missing_score + self.score_range = score_range + self.error_target = error_target + self.score_error_target = score_error_target or error_target + self.binary_cutoff = binary_cutoff + self.higher_is_better = higher_is_better + self.completed_alias = completed_alias + self.top_level_usage = top_level_usage + self.flow_resolver = flow_resolver + + async def evaluate(self, eval_input: Dict[str, Any]) -> Dict[str, Any]: + runner = self.flow_resolver() if self.flow_resolver else self.runner + return self.parse(await runner(timeout=self.timeout, **eval_input)) + + def skipped(self, reason: str, threshold: Optional[float] = None) -> Dict[str, Any]: + result = not_applicable_result( + self.result_key, reason, self.threshold if threshold is None else threshold + ) + if self.top_level_usage: + metadata = ModelEvaluatorBase._get_token_metadata({}) + result.update({f"{self.result_key}_{key}": value for key, value in metadata.items()}) + return result + + def parse(self, output: Dict[str, Any]) -> Dict[str, Any]: + if not isinstance(output, dict) or (not self.strict and not output): + self._invalid_output() + answer = output.get("llm_output", output if self.strict else "") + if not self.strict and isinstance(answer, str): + try: + parsed = json.loads(answer) + except (json.JSONDecodeError, TypeError): + parsed = None + if isinstance(parsed, dict) and parsed: + answer = parsed + reason = "" + properties = {} + score = self.missing_score + if isinstance(answer, dict) and (self.strict or answer): + if answer.get("status", "completed") == "skipped": + return self.skipped(answer.get("reason", "")) + score = answer.get("score", self.missing_score) + reason = answer.get("reason", "") + properties = answer.get("properties", {}) or {} + elif self.strict: + self._invalid_output() + elif isinstance(answer, str): + if self.result_key in PROMPT_BASED_REASON_EVALUATORS: + score, reason = parse_quality_evaluator_reason_score(answer) + else: + match = re.search(r"\d", answer) + if match: + score = float(match.group()) + if self.score_range: + minimum, maximum = self.score_range + if not score or not check_score_is_valid(score, minimum, maximum): + raise EvaluationException( + message=f"Invalid score value: {score}. Expected a number in range [{minimum}, {maximum}].", + internal_message="Invalid score value.", + category=ErrorCategory.FAILED_EXECUTION, + target=self.score_error_target, + blame=ErrorBlame.SYSTEM_ERROR, + ) + score = float(score) if self.strict or score is not None else math.nan + cutoff = self.threshold if self.binary_cutoff is None else self.binary_cutoff + if math.isnan(score) and self.binary_cutoff is None: + verdict = "unknown" + else: + passed = score >= cutoff if self.higher_is_better else score <= cutoff + verdict = "pass" if passed else "fail" + return self.format_result(score, verdict, reason, "completed", properties, output) + + def format_result( + self, score: Optional[float], verdict: str, reason: str, status: str, + properties: Dict[str, Any], output: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + metadata = ModelEvaluatorBase._get_token_metadata(output or {}) + properties.update(metadata) + result = { + self.result_key: score, + f"{self.result_key}_score": score, + f"{self.result_key}_passed": verdict == "pass" if verdict in ("pass", "fail", "unknown") else None, + f"{self.result_key}_result": verdict, + f"{self.result_key}_reason": reason, + f"{self.result_key}_status": status, + f"{self.result_key}_threshold": self.threshold, + f"{self.result_key}_properties": properties, + } + if self.completed_alias and status == "completed": + result[self.completed_alias] = score + if self.top_level_usage: + result.update({f"{self.result_key}_{key}": value for key, value in metadata.items()}) + return result + + def _invalid_output(self) -> None: + raise EvaluationException( + message="Evaluator returned invalid output.", + blame=ErrorBlame.SYSTEM_ERROR, + category=ErrorCategory.FAILED_EXECUTION, + target=self.error_target, + ) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py new file mode 100644 index 000000000000..40b1e8350983 --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py @@ -0,0 +1,546 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- + +"""Offline tests for shared decision-model runners, base selection, and evaluation strategy.""" + +import asyncio +import json +import threading +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock, patch + +import pytest +from azure.core.pipeline.policies import AsyncRedirectPolicy, AsyncRetryPolicy +from azure.core.pipeline.transport import AsyncHttpTransport, HttpResponse + +from azure.ai.evaluation import FluencyEvaluator +from azure.ai.evaluation._evaluators._common import EvaluatorBase, ModelEvaluatorBase, PromptyEvaluatorBase +from azure.ai.evaluation._evaluators._common import _model_runners +from azure.ai.evaluation._evaluators._common._decision_strategy import ( + BinaryScoring, + DecisionEvaluationStrategy, + LikertScoring, +) +from azure.ai.evaluation._evaluators._common._model_runners import ( + DecisionModelRunner, + PromptyModelRunner, + validate_noul_answer, + validate_score_answer, +) +from azure.ai.evaluation._exceptions import ErrorBlame, ErrorTarget, EvaluationException + +pytestmark = pytest.mark.unittest + +DECISION_CONFIG = { + "type": "decision", + "azure_endpoint": "https://example.services.ai.azure.com", + "azure_deployment": "Decision-1", + "api_key": "test-key-value", +} +CHAT_CONFIG = { + "azure_endpoint": "https://example.openai.azure.com", + "azure_deployment": "chat-deployment", + "api_key": "chat-key", +} + + +def score_answer(levels): + """A consistent five-level score answer for the given level probabilities.""" + return { + "type": "score", + "score": sum(index * value for index, value in enumerate(levels)), + "confidence": max(levels), + "probabilities": {str(index): value for index, value in enumerate(levels)}, + } + + +def decision_payload(status=0.95, result=None, model="microsoft-decision-1", usage=(120, 3), **answers): + payload = { + "model": model, + "answers": {"status": {"type": "noul", "noul": status}}, + "usage": {"input_tokens": usage[0], "output_tokens": usage[1]}, + } + if result is not None: + payload["answers"].update(result) + payload["answers"].update(answers) + return payload + + +def level_payload(level, status=0.95): + levels = [0.0] * 5 + levels[level] = 1.0 + return decision_payload(status, {"tool_call_accuracy": score_answer(levels)}) + + +class FakeResponse: + def __init__(self, status_code=200, body=b"{}"): + self.status_code = status_code + self._body = body + + def body(self): + return self._body + + +class FakeClient: + def __init__(self, response=None, delay=0.0): + self.response = response or FakeResponse() + self.delay = delay + self.calls = [] + self.policies = None + + def with_policies(self, **policies): + self.policies = policies + return self + + async def __aenter__(self): + return self + + async def __aexit__(self, *_): + return False + + async def post(self, url, json=None, headers=None, **options): # pylint: disable=redefined-outer-name + self.calls.append({"url": url, "json": json, "headers": dict(headers), "options": options}) + if self.delay: + await asyncio.sleep(self.delay) + return self.response + + +@pytest.fixture +def client(monkeypatch): + fake = FakeClient() + monkeypatch.setattr(_model_runners, "get_async_http_client", lambda: fake) + return fake + + +QUESTIONS = { + "status": {"type": "noul", "instructions": "Applicable?"}, + "tool_call_accuracy": {"type": "score", "instructions": "How accurate?", "criteria": ["a", "b", "c", "d", "e"]}, +} + + +def run(runner, **overrides): + arguments = {"state": {"query": "q"}, "questions": QUESTIONS, "timeout": 5} + arguments.update(overrides) + return asyncio.run(runner(**arguments)) + + +class TestDecisionModelRunnerConfiguration: + @pytest.mark.parametrize( + "changes", + [ + {"type": "azure_openai"}, + {"extra": "field"}, + {"azure_endpoint": None}, + {"azure_endpoint": " "}, + {"azure_deployment": " "}, + {"azure_deployment": None}, + {"api_key": " "}, + {"probability_threshold": 0.5}, + ], + ) + def test_invalid_configuration_is_a_user_error(self, changes): + config = {**DECISION_CONFIG, **changes} + with pytest.raises(EvaluationException) as raised: + DecisionModelRunner(config) + assert raised.value.blame == ErrorBlame.USER_ERROR + assert "secret" not in str(raised.value) + + @pytest.mark.parametrize("value", [True, "0.5", None, float("nan"), -0.1, 1.1]) + def test_invalid_probability_threshold(self, value): + with pytest.raises(EvaluationException): + DecisionModelRunner({**DECISION_CONFIG, "probability_threshold": value}, binary=True) + + @pytest.mark.parametrize("endpoint", [ + "https://example.services.ai.azure.com", + "https://example.services.ai.azure.com/", + "http://example.services.ai.azure.com", + "https://user:secret@example.services.ai.azure.com", + "https://example.services.ai.azure.com?x=1", + "https://example.services.ai.azure.com#fragment", + "https://example.services.ai.azure.com/api/projects/proj", + "not-a-url", + ]) + def test_endpoint_format_is_left_to_http_client(self, endpoint): + runner = DecisionModelRunner({**DECISION_CONFIG, "azure_endpoint": endpoint}) + assert runner._url == endpoint.rstrip("/") + "/providers/microsoft/v1/systemone" + + def test_binary_probability_threshold(self): + assert DecisionModelRunner(DECISION_CONFIG, binary=True).probability_threshold == 0.75 + runner = DecisionModelRunner({**DECISION_CONFIG, "probability_threshold": 0.7}, binary=True) + assert runner.probability_threshold == 0.7 + + def test_api_key_wins_over_credential(self): + runner = DecisionModelRunner(DECISION_CONFIG, credential=Mock()) + assert runner._credential is None and runner._api_key == DECISION_CONFIG["api_key"] + + +class TestDecisionModelRunnerRequests: + def test_request_shape_with_api_key(self, client): + client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) + result = run(DecisionModelRunner(DECISION_CONFIG, evaluator_name="Probe"), state={"query": "q"}) + call = client.calls[0] + assert call["url"] == "https://example.services.ai.azure.com/providers/microsoft/v1/systemone" + assert call["json"] == {"model": "Decision-1", "state": {"query": "q"}, "questions": QUESTIONS} + assert call["headers"]["api-key"] == "test-key-value" + assert "Authorization" not in call["headers"] + assert call["headers"]["x-ms-useragent"].endswith("(type=evaluator subtype=Probe)") + assert result["usage"] == {"input_tokens": 120, "output_tokens": 3} + + @pytest.mark.parametrize("asynchronous", [False, True]) + def test_credential_acquires_a_scoped_bearer_token(self, client, asynchronous): + client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) + token = SimpleNamespace(token="bearer-value") + credential = Mock() + credential.get_token = AsyncMock(return_value=token) if asynchronous else Mock(return_value=token) + config = {key: value for key, value in DECISION_CONFIG.items() if key != "api_key"} + run(DecisionModelRunner(config, credential)) + credential.get_token.assert_called_once_with("https://cognitiveservices.azure.com/.default") + headers = client.calls[0]["headers"] + assert headers["Authorization"] == "Bearer bearer-value" and "api-key" not in headers + + def test_credential_in_configuration(self, client): + client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) + credential = Mock(get_token=Mock(return_value=SimpleNamespace(token="from-config"))) + config = {key: value for key, value in DECISION_CONFIG.items() if key != "api_key"} + run(DecisionModelRunner({**config, "credential": credential})) + assert client.calls[0]["headers"]["Authorization"] == "Bearer from-config" + + def test_bounded_retries_without_redirects(self, client): + client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) + run(DecisionModelRunner(DECISION_CONFIG)) + retry, redirect = client.policies["retry_policy"], client.policies["redirect_policy"] + assert isinstance(retry, AsyncRetryPolicy) and retry.total_retries == 3 + assert isinstance(redirect, AsyncRedirectPolicy) and redirect.allow is False + + @pytest.mark.parametrize("status", _model_runners._DECISION_RETRY_STATUS_CODES) + def test_transient_post_response_is_retried_without_retry_after(self, monkeypatch, status): + from azure.ai.evaluation._http_utils import AsyncHttpPipeline + + class Response(HttpResponse): + def __init__(self, request, status_code): + super().__init__(request, None) + self.status_code = status_code + self.headers = {} + + def body(self): + return json.dumps(level_payload(4)).encode() + + class Transport(AsyncHttpTransport): + def __init__(self): + self.calls = 0 + + async def open(self): + pass + + async def close(self): + pass + + async def __aexit__(self, *args): + await self.close() + + async def send(self, request, **kwargs): + self.calls += 1 + return Response(request, status if self.calls == 1 else 200) + + transport = Transport() + monkeypatch.setattr( + _model_runners, "get_async_http_client", lambda: AsyncHttpPipeline(transport=transport) + ) + result = run(DecisionModelRunner(DECISION_CONFIG)) + assert transport.calls == 2 + assert result["model"] == "microsoft-decision-1" + + def test_default_credential_token_acquisition_runs_off_event_loop(self, client): + from azure.ai.evaluation._legacy._common._async_token_provider import AsyncAzureTokenProvider + + client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) + loop_thread = threading.get_ident() + token_threads = [] + + def get_token(*scopes, **kwargs): + token_threads.append(threading.get_ident()) + return SimpleNamespace(token="default-token") + + credential = Mock(get_token=get_token) + config = {key: value for key, value in DECISION_CONFIG.items() if key != "api_key"} + with patch.object(AsyncAzureTokenProvider, "_initialize_async", new=AsyncMock(return_value=credential)): + run(DecisionModelRunner(config)) + assert token_threads and all(token_thread != loop_thread for token_thread in token_threads) + assert client.calls[0]["headers"]["Authorization"] == "Bearer default-token" + + def test_default_credential_does_not_block_request_timeout(self, client): + from azure.ai.evaluation._legacy._common._async_token_provider import AsyncAzureTokenProvider + + release = threading.Event() + + def get_token(*scopes, **kwargs): + release.wait(timeout=2) + return SimpleNamespace(token="default-token") + + credential = Mock(get_token=get_token) + config = {key: value for key, value in DECISION_CONFIG.items() if key != "api_key"} + + async def request(): + try: + with pytest.raises(EvaluationException, match="timed out"): + await DecisionModelRunner(config)(state={}, questions=QUESTIONS, timeout=0.02) + finally: + release.set() + + with patch.object(AsyncAzureTokenProvider, "_initialize_async", new=AsyncMock(return_value=credential)): + asyncio.run(request()) + assert not client.calls + + @pytest.mark.parametrize( + "status, blame", + [(400, ErrorBlame.USER_ERROR), (401, ErrorBlame.USER_ERROR), (404, ErrorBlame.USER_ERROR), + (429, ErrorBlame.SYSTEM_ERROR), (503, ErrorBlame.SYSTEM_ERROR)], + ) + def test_http_failures_are_sanitized(self, client, status, blame): + client.response = FakeResponse(status, b"echoed test-key-value and secret detail") + with pytest.raises(EvaluationException) as raised: + run(DecisionModelRunner(DECISION_CONFIG)) + assert raised.value.blame == blame + assert str(status) in str(raised.value) + assert "test-key-value" not in str(raised.value) and "secret detail" not in str(raised.value) + + def test_transport_failure_is_sanitized(self, monkeypatch): + class Broken(FakeClient): + async def post(self, *_, **__): + raise ConnectionError("https://example.services.ai.azure.com test-key-value") + + monkeypatch.setattr(_model_runners, "get_async_http_client", lambda: Broken()) + with pytest.raises(EvaluationException) as raised: + run(DecisionModelRunner(DECISION_CONFIG)) + assert "test-key-value" not in str(raised.value) and "example.services" not in str(raised.value) + + def test_token_failure_is_a_user_error(self, client): + credential = Mock(get_token=Mock(side_effect=RuntimeError("token-secret"))) + config = {key: value for key, value in DECISION_CONFIG.items() if key != "api_key"} + with pytest.raises(EvaluationException) as raised: + run(DecisionModelRunner(config, credential)) + assert raised.value.blame == ErrorBlame.USER_ERROR and "token-secret" not in str(raised.value) + assert not client.calls + + def test_invalid_json(self, client): + client.response = FakeResponse(body=b"not json") + with pytest.raises(EvaluationException, match="invalid JSON"): + run(DecisionModelRunner(DECISION_CONFIG)) + + def test_timeout(self, client): + client.delay = 1 + client.response = FakeResponse(body=json.dumps(decision_payload()).encode()) + with pytest.raises(EvaluationException, match="timed out"): + run(DecisionModelRunner(DECISION_CONFIG), timeout=0.01) + + @pytest.mark.parametrize( + "mutate", + [ + lambda p: p.pop("answers"), + lambda p: p["answers"].pop("status"), + lambda p: p["answers"].update(extra={"type": "noul", "noul": 1}), + lambda p: p["answers"]["status"].update(type="score"), + lambda p: p.pop("usage"), + lambda p: p["usage"].update(input_tokens=-1), + lambda p: p["usage"].update(output_tokens=True), + lambda p: p.update(model=" "), + ], + ) + def test_structurally_invalid_responses(self, client, mutate): + payload = level_payload(4) + mutate(payload) + client.response = FakeResponse(body=json.dumps(payload).encode()) + with pytest.raises(EvaluationException, match="invalid response"): + run(DecisionModelRunner(DECISION_CONFIG)) + + def test_unsupported_question_type(self, client): + with pytest.raises(EvaluationException, match="not supported") as raised: + run(DecisionModelRunner(DECISION_CONFIG), questions={"team": {"type": "choice", "criteria": {}}}) + assert raised.value.blame == ErrorBlame.USER_ERROR + assert not client.calls + + +class TestAnswerValidation: + def test_noul(self): + assert validate_noul_answer({"type": "noul", "noul": 0.25}) == 0.25 + for bad in (True, "0.5", None, float("nan"), -0.01, 1.01): + with pytest.raises(ValueError): + validate_noul_answer({"type": "noul", "noul": bad}) + + def test_score_fields_do_not_require_consistency_checks(self): + levels = [0.1, 0.2, 0.3, 0.4, 0.5] + answer = score_answer(levels) + answer["score"] = 0 + assert validate_score_answer(answer, 5) == levels + + @pytest.mark.parametrize( + "mutate", + [ + lambda a: a["probabilities"].pop("4"), + lambda a: a["probabilities"].update({"5": 0.0}), + lambda a: a["probabilities"].update({"4": 1.5}), + lambda a: a.update(score=float("nan")), + lambda a: a.update(score=True), + lambda a: a.update(confidence=2), + ], + ) + def test_malformed_scores(self, mutate): + answer = score_answer([0, 0, 0, 0, 1.0]) + mutate(answer) + with pytest.raises(ValueError): + validate_score_answer(answer, 5) + + +class ProbeModelEvaluator(ModelEvaluatorBase): + _supports_decision_models = True + + async def _do_eval(self, eval_input): + return await self._model_runner(timeout=self._LLM_CALL_TIMEOUT, **eval_input) + + +class TestSharedBaseSelection: + def test_hierarchy(self): + assert issubclass(ModelEvaluatorBase, EvaluatorBase) + assert issubclass(PromptyEvaluatorBase, ModelEvaluatorBase) + + def test_decision_configuration_selects_the_decision_runner_without_loading_prompty(self): + with patch.object(_model_runners.AsyncPrompty, "load", side_effect=AssertionError("prompty loaded")): + evaluator = ProbeModelEvaluator(result_key="probe", model_config=DECISION_CONFIG) + assert isinstance(evaluator._model_runner, DecisionModelRunner) + assert evaluator._uses_decision_model and not hasattr(evaluator, "_flow") + + def test_decision_base_does_not_require_a_prompty_file(self): + with patch.object(_model_runners.AsyncPrompty, "load", side_effect=AssertionError("prompty loaded")): + evaluator = ProbeModelEvaluator(result_key="probe", model_config=DECISION_CONFIG) + assert evaluator._decision_model_runner is evaluator._model_runner + assert not hasattr(evaluator, "_flow") + + def test_chat_base_requires_a_prompty_file(self): + with pytest.raises(EvaluationException, match="prompty_file"): + ProbeModelEvaluator(result_key="probe", model_config=CHAT_CONFIG) + + def test_decision_runner_accessor_rejects_a_chat_runner(self): + with patch.object(_model_runners.AsyncPrompty, "load"): + evaluator = ProbeModelEvaluator(result_key="probe", prompty_file="probe.prompty", model_config=CHAT_CONFIG) + with pytest.raises(RuntimeError, match="not configured"): + _ = evaluator._decision_model_runner + + def test_replacing_legacy_flow_updates_runner_execution(self): + with patch.object(_model_runners.AsyncPrompty, "load"): + evaluator = ProbeModelEvaluator(result_key="probe", prompty_file="probe.prompty", model_config=CHAT_CONFIG) + flow = AsyncMock(return_value={"llm_output": {"score": 1, "reason": "replacement", "properties": {}}}) + evaluator._flow = flow + assert evaluator._prompty_model_runner.flow is flow + inputs = {"query": "q", "response": "r"} + assert asyncio.run(evaluator._do_eval(inputs)) == flow.return_value + flow.assert_awaited_once_with(timeout=evaluator._LLM_CALL_TIMEOUT, **inputs) + + @pytest.mark.parametrize("tag", ["Decision", "system_one", "azure_openai", None, ""]) + def test_other_tags_are_not_decision_configurations(self, tag): + assert not _model_runners.is_decision_model_config({**DECISION_CONFIG, "type": tag}) + with pytest.raises(EvaluationException): + DecisionModelRunner({**DECISION_CONFIG, "type": tag}) + + def test_chat_configuration_selects_the_prompty_runner_without_a_decision_runner(self): + flow = AsyncMock() + with patch.object(_model_runners.AsyncPrompty, "load", return_value=flow), patch.object( + _model_runners, "DecisionModelRunner", side_effect=AssertionError("decision runner built") + ): + evaluator = ProbeModelEvaluator(result_key="probe", prompty_file="probe.prompty", model_config=CHAT_CONFIG) + assert isinstance(evaluator._model_runner, PromptyModelRunner) + assert evaluator._flow is flow and not evaluator._uses_decision_model + + def test_prompty_only_evaluators_reject_a_decision_configuration(self): + with patch.object(_model_runners.AsyncPrompty, "load", side_effect=AssertionError("prompty loaded")): + with pytest.raises(EvaluationException) as raised: + FluencyEvaluator(DECISION_CONFIG) + assert raised.value.blame == ErrorBlame.USER_ERROR + assert "FluencyEvaluator" in str(raised.value) + + def test_wrapper_is_initialized_once(self): + original = EvaluatorBase.__init__ + initialized = [] + + def record(instance, **kwargs): + initialized.append(instance) + original(instance, **kwargs) + + with patch.object(EvaluatorBase, "__init__", record): + evaluator = ProbeModelEvaluator(result_key="probe", model_config=DECISION_CONFIG) + assert initialized == [evaluator] + + def test_prompty_load_parity(self): + flow = AsyncMock() + credential = object() + with patch.object(_model_runners.AsyncPrompty, "load", return_value=flow) as load: + evaluator = ProbeModelEvaluator( + result_key="probe", prompty_file="probe.prompty", model_config=CHAT_CONFIG, + credential=credential, threshold=4, + ) + kwargs = load.call_args.kwargs + assert kwargs["source"] == "probe.prompty" + assert kwargs["token_credential"] is credential and kwargs["is_reasoning_model"] is False + headers = kwargs["model"]["parameters"]["extra_headers"] + assert headers["x-ms-useragent"].endswith("(type=evaluator subtype=ProbeModelEvaluator)") + assert kwargs["model"]["configuration"]["api_version"] == "2024-02-15-preview" + assert evaluator._threshold == 4 and evaluator._flow is flow + + def test_inherited_helpers_remain_available_to_prompty_evaluators(self): + with patch.object(_model_runners.AsyncPrompty, "load"): + evaluator = FluencyEvaluator(CHAT_CONFIG) + assert isinstance(evaluator, PromptyEvaluatorBase) + assert evaluator._get_binary_result(5) == "pass" and evaluator._get_binary_result(float("nan")) == "unknown" + assert evaluator._get_token_metadata({"input_token_count": 3})["prompt_tokens"] == 3 + assert evaluator._return_not_applicable_result("why", 3)["fluency_status"] == "skipped" + + +def completion_payload(probability, status=0.95): + return decision_payload(status, {"task_completion": {"type": "noul", "noul": probability}}) + + +def strategy_with(payload, scoring, key="tool_call_accuracy", extra_properties=None): + runner = AsyncMock(return_value=payload) + runner.deployment = "Decision-1" + return DecisionEvaluationStrategy( + runner, + result_key=key, + error_target=ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR, + questions={"status": {"type": "noul"}, key: {"type": "score"}}, + scoring=scoring, + threshold=3, + timeout=5, + extra_properties=extra_properties, + ) + + +class TestSharedDecisionStrategy: + @pytest.mark.parametrize("probability, score", [(0.6, 1.0), (0.5999, 0.0)]) + def test_binary_scoring_uses_its_own_cutoff(self, probability, score): + strategy = strategy_with(completion_payload(probability), BinaryScoring(0.6), key="task_completion") + assert asyncio.run(strategy.evaluate({}))["task_completion"] == score + + @pytest.mark.parametrize("minimum, score", [(0, 2.0), (1, 3.0)]) + def test_likert_scoring_offsets_the_level_by_the_scale_minimum(self, minimum, score): + strategy = strategy_with(level_payload(2), LikertScoring(5, minimum=minimum)) + assert asyncio.run(strategy.evaluate({}))["tool_call_accuracy"] == score + + def test_likert_scoring_selects_the_lowest_tied_level(self): + payload = decision_payload(0.95, {"tool_call_accuracy": score_answer([0.0, 0.5, 0.0, 0.5, 0.0])}) + assert asyncio.run(strategy_with(payload, LikertScoring(5)).evaluate({}))["tool_call_accuracy"] == 2.0 + + def test_invalid_answer_reports_the_evaluator_error_target(self): + payload = level_payload(4) + payload["answers"]["tool_call_accuracy"]["probabilities"]["0"] = 1.5 + with pytest.raises(EvaluationException) as raised: + asyncio.run(strategy_with(payload, LikertScoring(5)).evaluate({})) + assert raised.value.target == ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR + + def test_extra_properties_are_added_with_the_decision_provenance(self): + strategy = strategy_with( + level_payload(4), LikertScoring(5), extra_properties=lambda eval_input: {"calls": len(eval_input["calls"])} + ) + properties = asyncio.run(strategy.evaluate({"calls": [1, 2]}))["tool_call_accuracy_properties"] + assert properties["calls"] == 2 and properties["decision_model"]["deployment"] == "Decision-1" + + diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_strategy.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_strategy.py new file mode 100644 index 000000000000..0b594c63d397 --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_strategy.py @@ -0,0 +1,134 @@ +import asyncio +import math +from unittest.mock import AsyncMock + +import pytest + +from azure.ai.evaluation._evaluators._common._prompty_strategy import PromptyEvaluationStrategy +from azure.ai.evaluation._exceptions import EvaluationException, ErrorTarget + + +def strategy(**kwargs): + options = {"result_key": "metric", "threshold": 3, "timeout": 17} + options.update(kwargs) + return PromptyEvaluationStrategy(AsyncMock(), **options) + + +@pytest.mark.parametrize("answer,expected,reason", [ + ({"score": 4, "reason": "good"}, 4, "good"), + ('{"score": 4, "reason": "good"}', 4, "good"), + ("malformed JSON { score 4", 4, ""), + ("the score is 2", 2, ""), + ("[] score 2", 2, ""), + (None, math.nan, ""), + ({"score": None}, math.nan, ""), + ({}, math.nan, ""), + ("no score", math.nan, ""), +]) +def test_legacy_parser(answer, expected, reason): + result = strategy(strict=False, missing_score=math.nan).parse({"llm_output": answer}) + if math.isnan(expected): + assert math.isnan(result["metric"]) + assert result["metric_result"] == "unknown" + assert result["metric_passed"] is False + else: + assert result["metric"] == expected + assert result["metric_reason"] == reason + + +def test_legacy_quality_xml(): + result = strategy(result_key="fluency", strict=False, missing_score=math.nan).parse({ + "llm_output": " Clear prose 4", + }) + assert result["fluency"] == 4 + assert result["fluency_reason"] == "Clear prose" + + +@pytest.mark.parametrize("higher,score,expected", [ + (True, 2, "fail"), (True, 3, "pass"), (True, 4, "pass"), + (False, 2, "pass"), (False, 3, "pass"), (False, 4, "fail"), +]) +def test_threshold_direction(higher, score, expected): + assert strategy(higher_is_better=higher).parse({"score": score})["metric_result"] == expected + + +@pytest.mark.parametrize("output", [ + None, [], "text", {"llm_output": "{\"score\":4}"}, {"llm_output": []}, {"llm_output": None}, +]) +def test_strict_rejects_non_dict_answer(output): + with pytest.raises(EvaluationException, match="Evaluator returned invalid output") as error: + strategy(error_target=ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR).parse(output) + assert error.value.target == ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR + + +@pytest.mark.parametrize("score", [None, 0, -1, 6, "bad", math.nan, math.inf]) +def test_score_range_errors(score): + with pytest.raises(EvaluationException, match="Invalid score value") as error: + strategy(score_range=(1, 5), error_target=ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR).parse({"score": score}) + assert error.value.target == ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR + + +def test_missing_completion_score_and_binary_rule(): + completion = strategy(binary_cutoff=1, threshold=7) + assert completion.parse({})["metric"] == 0 + assert completion.parse({"score": 1})["metric_result"] == "pass" + with pytest.raises(ValueError): + completion.parse({"score": "bad"}) + + +@pytest.mark.parametrize("strict", [True, False]) +def test_skipped_bypasses_score_validation(strict): + result = strategy(strict=strict, score_range=(1, 5)).parse({ + "llm_output": {"score": "bad", "status": "skipped", "reason": "not relevant"}, + }) + assert result == { + "metric": None, "metric_score": None, "metric_passed": None, + "metric_result": "not_applicable", "metric_reason": "Not applicable: not relevant", + "metric_status": "skipped", "metric_threshold": 3, "metric_properties": None, + } + + +@pytest.mark.parametrize("key,alias,usage", [ + ("task_completion", None, False), + ("tool_call_accuracy", "gpt_tool_call_accuracy", False), + ("task_completion", None, True), + ("tool_call_accuracy", None, False), +]) +def test_exact_chat_schemas(key, alias, usage): + configured = strategy(result_key=key, completed_alias=alias, top_level_usage=usage) + output = {"llm_output": {"score": 4, "properties": {"custom": "kept"}}, "input_token_count": 9} + completed = configured.parse(output) + metadata = { + "prompt_tokens": 9, "completion_tokens": 0, "total_tokens": 0, "finish_reason": "", + "model": "", "sample_input": "", "sample_output": "", + } + expected_keys = {key} | {f"{key}_{suffix}" for suffix in ( + "score", "passed", "result", "reason", "status", "threshold", "properties", + )} + assert completed[f"{key}_properties"] == {"custom": "kept", **metadata} + if alias: + expected_keys.add(alias) + assert completed[alias] == 4 + if usage: + expected_keys.update(f"{key}_{name}" for name in metadata) + assert completed[f"{key}_prompt_tokens"] == 9 + assert set(completed) == expected_keys + skipped = configured.parse({"llm_output": {"status": "skipped"}, "input_token_count": 9}) + assert set(skipped) == expected_keys - ({alias} if alias else set()) + if usage: + assert skipped[f"{key}_prompt_tokens"] == 0 + + +def test_runtime_flow_resolution(): + original = AsyncMock(side_effect=AssertionError("stale flow")) + current = original + configured = strategy(flow_resolver=lambda: current) + current = AsyncMock(return_value={"score": 4}) + assert asyncio.run(configured.evaluate({"query": "q"}))["metric"] == 4 + current.assert_awaited_once_with(timeout=17, query="q") + original.assert_not_awaited() + + +def test_legacy_empty_envelope_is_invalid(): + with pytest.raises(EvaluationException, match="Evaluator returned invalid output"): + strategy(strict=False).parse({}) From 925943d2efb3b5fcf02a43f4aed6a2227070466b Mon Sep 17 00:00:00 2001 From: Salma Elshafey Date: Mon, 12 Oct 2026 13:35:12 +0300 Subject: [PATCH 02/24] ignore noul word --- sdk/evaluation/cspell.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdk/evaluation/cspell.yaml b/sdk/evaluation/cspell.yaml index 8279e22c0eef..4861b96360db 100644 --- a/sdk/evaluation/cspell.yaml +++ b/sdk/evaluation/cspell.yaml @@ -32,6 +32,7 @@ words: - nbformat - ncols - ndcg + - noul - ollama - onedp - otel @@ -62,6 +63,8 @@ words: - xpia ignorePaths: - sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/onedp/models/_enums.py + - sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py + - sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py - sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_groundedness/groundedness_with_query.prompty - sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_intent_resolution/intent_resolution.prompty - sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_retrieval/retrieval.prompty From cd99addac973259e9e732bc69aebf6bd9bd1aabf Mon Sep 17 00:00:00 2001 From: Salma Elshafey Date: Mon, 12 Oct 2026 14:45:52 +0300 Subject: [PATCH 03/24] run black --- .../_evaluators/_common/_model_runners.py | 4 +- .../_evaluators/_common/_prompty_strategy.py | 13 +- .../unittests/test_decision_model_support.py | 45 ++++--- .../tests/unittests/test_prompty_strategy.py | 120 ++++++++++++------ 4 files changed, 117 insertions(+), 65 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py index 193ebdf3fa99..3491e822b464 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py @@ -210,9 +210,7 @@ def deployment(self) -> str: """The deployment name sent as the request's model.""" return self._deployment - async def __call__( - self, *, state: Any, questions: Dict[str, Dict[str, Any]], timeout: float - ) -> Dict[str, Any]: + async def __call__(self, *, state: Any, questions: Dict[str, Dict[str, Any]], timeout: float) -> Dict[str, Any]: """Request the decisions for ``questions`` about ``state`` and return the validated response. :keyword state: The prepared evidence, as text or JSON-serializable data. diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_prompty_strategy.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_prompty_strategy.py index 93ad2eed104e..9e2516003d53 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_prompty_strategy.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_prompty_strategy.py @@ -48,9 +48,7 @@ async def evaluate(self, eval_input: Dict[str, Any]) -> Dict[str, Any]: return self.parse(await runner(timeout=self.timeout, **eval_input)) def skipped(self, reason: str, threshold: Optional[float] = None) -> Dict[str, Any]: - result = not_applicable_result( - self.result_key, reason, self.threshold if threshold is None else threshold - ) + result = not_applicable_result(self.result_key, reason, self.threshold if threshold is None else threshold) if self.top_level_usage: metadata = ModelEvaluatorBase._get_token_metadata({}) result.update({f"{self.result_key}_{key}": value for key, value in metadata.items()}) @@ -105,8 +103,13 @@ def parse(self, output: Dict[str, Any]) -> Dict[str, Any]: return self.format_result(score, verdict, reason, "completed", properties, output) def format_result( - self, score: Optional[float], verdict: str, reason: str, status: str, - properties: Dict[str, Any], output: Optional[Dict[str, Any]] = None, + self, + score: Optional[float], + verdict: str, + reason: str, + status: str, + properties: Dict[str, Any], + output: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: metadata = ModelEvaluatorBase._get_token_metadata(output or {}) properties.update(metadata) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py index 40b1e8350983..879a366c8417 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py @@ -151,16 +151,19 @@ def test_invalid_probability_threshold(self, value): with pytest.raises(EvaluationException): DecisionModelRunner({**DECISION_CONFIG, "probability_threshold": value}, binary=True) - @pytest.mark.parametrize("endpoint", [ - "https://example.services.ai.azure.com", - "https://example.services.ai.azure.com/", - "http://example.services.ai.azure.com", - "https://user:secret@example.services.ai.azure.com", - "https://example.services.ai.azure.com?x=1", - "https://example.services.ai.azure.com#fragment", - "https://example.services.ai.azure.com/api/projects/proj", - "not-a-url", - ]) + @pytest.mark.parametrize( + "endpoint", + [ + "https://example.services.ai.azure.com", + "https://example.services.ai.azure.com/", + "http://example.services.ai.azure.com", + "https://user:secret@example.services.ai.azure.com", + "https://example.services.ai.azure.com?x=1", + "https://example.services.ai.azure.com#fragment", + "https://example.services.ai.azure.com/api/projects/proj", + "not-a-url", + ], + ) def test_endpoint_format_is_left_to_http_client(self, endpoint): runner = DecisionModelRunner({**DECISION_CONFIG, "azure_endpoint": endpoint}) assert runner._url == endpoint.rstrip("/") + "/providers/microsoft/v1/systemone" @@ -244,9 +247,7 @@ async def send(self, request, **kwargs): return Response(request, status if self.calls == 1 else 200) transport = Transport() - monkeypatch.setattr( - _model_runners, "get_async_http_client", lambda: AsyncHttpPipeline(transport=transport) - ) + monkeypatch.setattr(_model_runners, "get_async_http_client", lambda: AsyncHttpPipeline(transport=transport)) result = run(DecisionModelRunner(DECISION_CONFIG)) assert transport.calls == 2 assert result["model"] == "microsoft-decision-1" @@ -294,8 +295,13 @@ async def request(): @pytest.mark.parametrize( "status, blame", - [(400, ErrorBlame.USER_ERROR), (401, ErrorBlame.USER_ERROR), (404, ErrorBlame.USER_ERROR), - (429, ErrorBlame.SYSTEM_ERROR), (503, ErrorBlame.SYSTEM_ERROR)], + [ + (400, ErrorBlame.USER_ERROR), + (401, ErrorBlame.USER_ERROR), + (404, ErrorBlame.USER_ERROR), + (429, ErrorBlame.SYSTEM_ERROR), + (503, ErrorBlame.SYSTEM_ERROR), + ], ) def test_http_failures_are_sanitized(self, client, status, blame): client.response = FakeResponse(status, b"echoed test-key-value and secret detail") @@ -475,8 +481,11 @@ def test_prompty_load_parity(self): credential = object() with patch.object(_model_runners.AsyncPrompty, "load", return_value=flow) as load: evaluator = ProbeModelEvaluator( - result_key="probe", prompty_file="probe.prompty", model_config=CHAT_CONFIG, - credential=credential, threshold=4, + result_key="probe", + prompty_file="probe.prompty", + model_config=CHAT_CONFIG, + credential=credential, + threshold=4, ) kwargs = load.call_args.kwargs assert kwargs["source"] == "probe.prompty" @@ -542,5 +551,3 @@ def test_extra_properties_are_added_with_the_decision_provenance(self): ) properties = asyncio.run(strategy.evaluate({"calls": [1, 2]}))["tool_call_accuracy_properties"] assert properties["calls"] == 2 and properties["decision_model"]["deployment"] == "Decision-1" - - diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_strategy.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_strategy.py index 0b594c63d397..184919e09b38 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_strategy.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_strategy.py @@ -14,17 +14,20 @@ def strategy(**kwargs): return PromptyEvaluationStrategy(AsyncMock(), **options) -@pytest.mark.parametrize("answer,expected,reason", [ - ({"score": 4, "reason": "good"}, 4, "good"), - ('{"score": 4, "reason": "good"}', 4, "good"), - ("malformed JSON { score 4", 4, ""), - ("the score is 2", 2, ""), - ("[] score 2", 2, ""), - (None, math.nan, ""), - ({"score": None}, math.nan, ""), - ({}, math.nan, ""), - ("no score", math.nan, ""), -]) +@pytest.mark.parametrize( + "answer,expected,reason", + [ + ({"score": 4, "reason": "good"}, 4, "good"), + ('{"score": 4, "reason": "good"}', 4, "good"), + ("malformed JSON { score 4", 4, ""), + ("the score is 2", 2, ""), + ("[] score 2", 2, ""), + (None, math.nan, ""), + ({"score": None}, math.nan, ""), + ({}, math.nan, ""), + ("no score", math.nan, ""), + ], +) def test_legacy_parser(answer, expected, reason): result = strategy(strict=False, missing_score=math.nan).parse({"llm_output": answer}) if math.isnan(expected): @@ -37,24 +40,41 @@ def test_legacy_parser(answer, expected, reason): def test_legacy_quality_xml(): - result = strategy(result_key="fluency", strict=False, missing_score=math.nan).parse({ - "llm_output": " Clear prose 4", - }) + result = strategy(result_key="fluency", strict=False, missing_score=math.nan).parse( + { + "llm_output": " Clear prose 4", + } + ) assert result["fluency"] == 4 assert result["fluency_reason"] == "Clear prose" -@pytest.mark.parametrize("higher,score,expected", [ - (True, 2, "fail"), (True, 3, "pass"), (True, 4, "pass"), - (False, 2, "pass"), (False, 3, "pass"), (False, 4, "fail"), -]) +@pytest.mark.parametrize( + "higher,score,expected", + [ + (True, 2, "fail"), + (True, 3, "pass"), + (True, 4, "pass"), + (False, 2, "pass"), + (False, 3, "pass"), + (False, 4, "fail"), + ], +) def test_threshold_direction(higher, score, expected): assert strategy(higher_is_better=higher).parse({"score": score})["metric_result"] == expected -@pytest.mark.parametrize("output", [ - None, [], "text", {"llm_output": "{\"score\":4}"}, {"llm_output": []}, {"llm_output": None}, -]) +@pytest.mark.parametrize( + "output", + [ + None, + [], + "text", + {"llm_output": '{"score":4}'}, + {"llm_output": []}, + {"llm_output": None}, + ], +) def test_strict_rejects_non_dict_answer(output): with pytest.raises(EvaluationException, match="Evaluator returned invalid output") as error: strategy(error_target=ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR).parse(output) @@ -78,33 +98,57 @@ def test_missing_completion_score_and_binary_rule(): @pytest.mark.parametrize("strict", [True, False]) def test_skipped_bypasses_score_validation(strict): - result = strategy(strict=strict, score_range=(1, 5)).parse({ - "llm_output": {"score": "bad", "status": "skipped", "reason": "not relevant"}, - }) + result = strategy(strict=strict, score_range=(1, 5)).parse( + { + "llm_output": {"score": "bad", "status": "skipped", "reason": "not relevant"}, + } + ) assert result == { - "metric": None, "metric_score": None, "metric_passed": None, - "metric_result": "not_applicable", "metric_reason": "Not applicable: not relevant", - "metric_status": "skipped", "metric_threshold": 3, "metric_properties": None, + "metric": None, + "metric_score": None, + "metric_passed": None, + "metric_result": "not_applicable", + "metric_reason": "Not applicable: not relevant", + "metric_status": "skipped", + "metric_threshold": 3, + "metric_properties": None, } -@pytest.mark.parametrize("key,alias,usage", [ - ("task_completion", None, False), - ("tool_call_accuracy", "gpt_tool_call_accuracy", False), - ("task_completion", None, True), - ("tool_call_accuracy", None, False), -]) +@pytest.mark.parametrize( + "key,alias,usage", + [ + ("task_completion", None, False), + ("tool_call_accuracy", "gpt_tool_call_accuracy", False), + ("task_completion", None, True), + ("tool_call_accuracy", None, False), + ], +) def test_exact_chat_schemas(key, alias, usage): configured = strategy(result_key=key, completed_alias=alias, top_level_usage=usage) output = {"llm_output": {"score": 4, "properties": {"custom": "kept"}}, "input_token_count": 9} completed = configured.parse(output) metadata = { - "prompt_tokens": 9, "completion_tokens": 0, "total_tokens": 0, "finish_reason": "", - "model": "", "sample_input": "", "sample_output": "", + "prompt_tokens": 9, + "completion_tokens": 0, + "total_tokens": 0, + "finish_reason": "", + "model": "", + "sample_input": "", + "sample_output": "", + } + expected_keys = {key} | { + f"{key}_{suffix}" + for suffix in ( + "score", + "passed", + "result", + "reason", + "status", + "threshold", + "properties", + ) } - expected_keys = {key} | {f"{key}_{suffix}" for suffix in ( - "score", "passed", "result", "reason", "status", "threshold", "properties", - )} assert completed[f"{key}_properties"] == {"custom": "kept", **metadata} if alias: expected_keys.add(alias) From a81f06763595d7b77147b4704916996d06c1ec57 Mon Sep 17 00:00:00 2001 From: salma-elshafey Date: Tue, 13 Oct 2026 12:43:18 +0300 Subject: [PATCH 04/24] Add bounded Decision execution policy and retained request receipts Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e564c5c8-480e-48e7-a9e0-44450f774b5e --- .../_evaluators/_common/__init__.py | 3 + .../_evaluators/_common/_base_model_eval.py | 18 +- .../_common/_decision_execution.py | 81 +++ .../_evaluators/_common/_decision_strategy.py | 36 +- .../_evaluators/_common/_model_runners.py | 198 ++++++- .../unittests/test_decision_model_support.py | 510 +++++++++++++++++- 6 files changed, 814 insertions(+), 32 deletions(-) create mode 100644 sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_execution.py diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/__init__.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/__init__.py index aa848b355e25..fd90b6dd5bbf 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/__init__.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/__init__.py @@ -4,6 +4,7 @@ from ._base_eval import EvaluatorBase from ._base_model_eval import ModelEvaluatorBase +from ._decision_execution import DecisionExecutionPolicy, DecisionRequestObservation from ._base_prompty_eval import PromptyEvaluatorBase from ._base_rai_svc_eval import RaiServiceEvaluatorBase from ._base_multi_eval import MultiEvaluatorBase @@ -12,6 +13,8 @@ __all__ = [ "EvaluatorBase", "ModelEvaluatorBase", + "DecisionExecutionPolicy", + "DecisionRequestObservation", "PromptyEvaluatorBase", "RaiServiceEvaluatorBase", "MultiEvaluatorBase", diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py index 9636dd181533..7b500cae49e8 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py @@ -9,6 +9,7 @@ from azure.ai.evaluation._constants import EVALUATION_PASS_FAIL_MAPPING from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget from . import EvaluatorBase +from ._decision_execution import DecisionExecutionPolicy from ._model_runners import ( DecisionModelRunner, PromptyModelRunner, @@ -48,6 +49,9 @@ class ModelEvaluatorBase(EvaluatorBase[T]): :keyword is_reasoning_model: This parameter is in preview. If True, updates the config parameters in prompty file based on reasoning models. Defaults to False. :type is_reasoning_model: bool + :keyword decision_execution_policy: Optional transport-only limits and receipt callbacks for a decision runner. + Rejected for Prompty evaluators; omitted policies preserve existing Prompty behavior. + :paramtype decision_execution_policy: Optional[DecisionExecutionPolicy] """ _LLM_CALL_TIMEOUT = 600 @@ -67,6 +71,7 @@ def __init__( eval_last_turn: bool = False, threshold: int = 3, credential: Optional[TokenCredential] = None, + decision_execution_policy: Optional[DecisionExecutionPolicy] = None, _higher_is_better: bool = False, **kwargs, ) -> None: @@ -88,9 +93,20 @@ def __init__( target=ErrorTarget.EVALUATE, ) self._model_runner: Union[DecisionModelRunner, PromptyModelRunner] = DecisionModelRunner( - model_config, credential, binary=self._binary_decision, evaluator_name=subclass_name + model_config, + credential, + binary=self._binary_decision, + evaluator_name=subclass_name, + execution_policy=decision_execution_policy, ) else: + if decision_execution_policy is not None: + raise EvaluationException( + message="A decision_execution_policy is only supported by decision model evaluators.", + blame=ErrorBlame.USER_ERROR, + category=ErrorCategory.INVALID_VALUE, + target=ErrorTarget.EVALUATE, + ) if not prompty_file: raise EvaluationException( message="A prompty_file is required for a chat model evaluator.", diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_execution.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_execution.py new file mode 100644 index 000000000000..56d536f317d5 --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_execution.py @@ -0,0 +1,81 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- + +"""Transport-only execution settings for hosted decision evaluators.""" + +import math +from dataclasses import dataclass +from typing import Any, Callable, Dict, Literal, Optional, Tuple + +from ..._exceptions import EvaluationException + + +@dataclass(frozen=True) +class DecisionRequestObservation: + """A sanitized per-attempt receipt. Unknown usage and model values remain ``None``.""" + + attempt: int + outcome: Literal["response", "error", "timeout", "cancelled", "request_rejected"] + http_status: Optional[int] = None + input_tokens: Optional[int] = None + output_tokens: Optional[int] = None + model: Optional[str] = None + + +@dataclass(frozen=True) +class DecisionExecutionPolicy: + """Internal hosted execution policy, separate from the model configuration. + + ``timeout_seconds`` bounds each transport connect/read; ``deadline_seconds`` caps the entire + operation, including authentication and azure-core retries. Request budgets measure compact UTF-8 JSON, + not estimated tokens. ``max_state_question_bytes`` bounds the largest ``{"state", "question"}`` pair. + ``max_response_bytes`` limits the body accepted for JSON parsing, not transport buffering or allocation. + Cancellation is polled every 50 ms, including during authentication and retry backoff. + Cancellation and deadlines stop awaiting authentication, but cannot forcibly interrupt a synchronous + credential call already running in a worker thread or bound that thread's shutdown time. + The synchronous observer receives transport receipts only; callback failures are explicit errors. + """ + + timeout_seconds: Optional[float] = None + deadline_seconds: Optional[float] = None + max_retries: int = 3 + max_request_bytes: Optional[int] = None + max_state_question_bytes: Optional[int] = None + max_response_bytes: Optional[int] = None + cancellation_requested: Optional[Callable[[], bool]] = None + observer: Optional[Callable[[DecisionRequestObservation], None]] = None + + def __post_init__(self) -> None: + for name in ("timeout_seconds", "deadline_seconds"): + value = getattr(self, name) + if value is not None and ( + isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value <= 0 + ): + raise ValueError(f"{name} must be a positive finite number.") + for name in ("max_request_bytes", "max_state_question_bytes", "max_response_bytes"): + value = getattr(self, name) + if value is not None and (isinstance(value, bool) or not isinstance(value, int) or value <= 0): + raise ValueError(f"{name} must be a positive integer.") + if isinstance(self.max_retries, bool) or not isinstance(self.max_retries, int) or self.max_retries < 0: + raise ValueError("max_retries must be a nonnegative integer.") + for name in ("cancellation_requested", "observer"): + value = getattr(self, name) + if value is not None and not callable(value): + raise ValueError(f"{name} must be callable.") + + +class DecisionEvaluationException(EvaluationException): + """An execution or interpretation error retaining only known, sanitized decision metadata.""" + + def __init__( + self, + message: str, + *, + observations: Tuple[DecisionRequestObservation, ...] = (), + properties: Optional[Dict[str, Any]] = None, + **kwargs: Any, + ) -> None: + self.observations = tuple(observations) + self.properties = properties + super().__init__(message, **kwargs) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py index 3f4c171565c8..48412071fe02 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py @@ -13,8 +13,9 @@ from dataclasses import dataclass from typing import Any, Callable, Dict, List, Optional, Tuple, TypeVar, Union -from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget +from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget from ._base_model_eval import ModelEvaluatorBase, not_applicable_result +from ._decision_execution import DecisionEvaluationException from ._model_runners import DecisionModelRunner, validate_noul_answer, validate_score_answer STATUS_CUTOFF = 0.5 @@ -26,7 +27,7 @@ def _checked_value(validate: Callable[..., T], error_target: ErrorTarget, *args: return validate(*args) except ValueError as exc: message = f"The decision model returned an invalid response: {exc}" - raise EvaluationException( + raise DecisionEvaluationException( message=message, internal_message=message, blame=ErrorBlame.SYSTEM_ERROR, @@ -158,10 +159,6 @@ def __init__( async def evaluate(self, eval_input: Dict[str, Any]) -> Dict[str, Any]: key = self.result_key response = await self.runner(state=dict(eval_input), questions=self.questions, timeout=self.timeout) - skip_reason = decision_skip_reason(response, STATUS_CUTOFF, self.error_target) - if skip_reason: - return not_applicable_result(key, skip_reason, self.threshold) - score, reason, details = self.scoring.interpret(response, key, self.error_target) properties = self.extra_properties(eval_input) if self.extra_properties else {} properties["decision_model"] = { "deployment": self.runner.deployment, @@ -169,13 +166,32 @@ async def evaluate(self, eval_input: Dict[str, Any]) -> Dict[str, Any]: "rubric_hash": _digest(self.questions[key]), "status_rubric_hash": _digest(self.questions["status"]), "native_status_answer": response["answers"]["status"], - "status_completion_probability": decision_probability(response, "status", self.error_target), "status_probability_threshold": STATUS_CUTOFF, "reason_type": "templated", - **details, - "native_answer": response["answers"][key], } - properties.update(ModelEvaluatorBase._get_token_metadata(decision_token_counts(response))) + tokens = decision_token_counts(response) + properties.update( + prompt_tokens=tokens["input_token_count"], + completion_tokens=tokens["output_token_count"], + total_tokens=tokens["total_token_count"], + model=tokens["model_id"], + ) + try: + skip_reason = decision_skip_reason(response, STATUS_CUTOFF, self.error_target) + properties["decision_model"]["status_completion_probability"] = decision_probability( + response, "status", self.error_target + ) + if skip_reason: + result = not_applicable_result(key, skip_reason, self.threshold) + result[f"{key}_properties"] = properties + return result + score, reason, details = self.scoring.interpret(response, key, self.error_target) + except DecisionEvaluationException as exc: + exc.properties = properties + raise + properties["decision_model"].update(details) + properties["decision_model"]["native_answer"] = response["answers"][key] + properties.update(ModelEvaluatorBase._get_token_metadata(tokens)) passed = self.scoring.passes(score, self.threshold) return { key: score, diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py index 3491e822b464..75c19f9031fc 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py @@ -13,13 +13,14 @@ import json import math import os -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional -from azure.core.pipeline.policies import AsyncRedirectPolicy, AsyncRetryPolicy +from azure.core.pipeline.policies import AsyncRedirectPolicy, AsyncRetryPolicy, CustomHookPolicy from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException from ..._common.utils import construct_prompty_model_config, validate_model_config from ..._http_utils import get_async_http_client +from ._decision_execution import DecisionEvaluationException, DecisionExecutionPolicy, DecisionRequestObservation if os.getenv("AI_EVALS_USE_PF_PROMPTY", "false").lower() == "true": from promptflow.core._flow import AsyncPrompty @@ -102,8 +103,8 @@ def _invalid_config(message: str) -> EvaluationException: ) -def _failed(message: str, blame: ErrorBlame = ErrorBlame.SYSTEM_ERROR) -> EvaluationException: - return EvaluationException( +def _failed(message: str, blame: ErrorBlame = ErrorBlame.SYSTEM_ERROR) -> DecisionEvaluationException: + return DecisionEvaluationException( message=message, internal_message=message, blame=blame, @@ -154,6 +155,31 @@ def validate_score_answer(answer: Dict[str, Any], levels: int) -> List[float]: return distribution +class _DecisionRetryPolicy(AsyncRetryPolicy): + def increment(self, settings: Dict[str, Any], response: Any = None, error: Optional[Exception] = None) -> bool: + # EvaluationException is an AzureError, but failed callbacks must not replay an already billed request. + if isinstance(error, EvaluationException): + return False + return super().increment(settings, response=response, error=error) + + +class _DecisionObservationPolicy(CustomHookPolicy): + def __init__(self, runner: "DecisionModelRunner", observations: List[DecisionRequestObservation]) -> None: + super().__init__() + self.runner = runner + self.observations = observations + self.attempt = 0 + + def on_request(self, request: Any) -> None: + self.attempt += 1 + + def on_response(self, request: Any, response: Any) -> None: + self.runner._observe(self.runner._response_observation(response.http_response, self.attempt), self.observations) + + def on_exception(self, request: Any) -> None: + self.runner._observe(DecisionRequestObservation(self.attempt, "error"), self.observations) + + class DecisionModelRunner: """Sends prepared evidence and typed questions to a Microsoft-Decision-1 deployment in Foundry. @@ -171,7 +197,11 @@ def __init__( *, binary: bool = False, evaluator_name: str = "DecisionModelRunner", + execution_policy: Optional[DecisionExecutionPolicy] = None, ) -> None: + if execution_policy is not None and not isinstance(execution_policy, DecisionExecutionPolicy): + raise _invalid_config("'execution_policy' must be a DecisionExecutionPolicy.") + self._execution_policy = execution_policy or DecisionExecutionPolicy() if not is_decision_model_config(model_config): raise _invalid_config(f"A decision model configuration must be a dict with type '{DECISION_MODEL_TYPE}'.") unexpected = set(model_config) - _DECISION_CONFIG_KEYS - _IGNORED_CONFIG_KEYS @@ -225,18 +255,100 @@ async def __call__(self, *, state: Any, questions: Dict[str, Dict[str, Any]], ti for question in questions.values(): if question.get("type") not in ("noul", "score"): raise _invalid_config(f"Decision question type '{question.get('type')}' is not supported.") + policy = self._execution_policy + observations: List[DecisionRequestObservation] = [] + observation_policy = _DecisionObservationPolicy(self, observations) + try: + body = {"model": self._deployment, "state": state, "questions": questions} + try: + content = json.dumps(body, ensure_ascii=False, separators=(",", ":"), allow_nan=False).encode("utf-8") + state_question_bytes = max( + ( + len( + json.dumps( + {"state": state, "question": question}, + ensure_ascii=False, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + ) + for question in questions.values() + ), + default=0, + ) + except (ValueError, TypeError, OverflowError, UnicodeError): + self._observe(DecisionRequestObservation(0, "request_rejected"), observations) + raise _failed( + "Decision evidence must be finite JSON-serializable data.", ErrorBlame.USER_ERROR + ) from None + if (policy.max_request_bytes is not None and len(content) > policy.max_request_bytes) or ( + policy.max_state_question_bytes is not None and state_question_bytes > policy.max_state_question_bytes + ): + self._observe(DecisionRequestObservation(0, "request_rejected"), observations) + raise _failed("The decision request exceeds the configured byte budget.", ErrorBlame.USER_ERROR) + deadline = min(timeout, policy.deadline_seconds) if policy.deadline_seconds is not None else timeout + try: + payload = await asyncio.wait_for(self._await_request(content, observation_policy), timeout=deadline) + except asyncio.TimeoutError: + self._observe(DecisionRequestObservation(observation_policy.attempt, "timeout"), observations) + raise _failed("The decision model request timed out.") from None + try: + return self._validate_response(payload, questions) + except ValueError as exc: + raise _failed(f"The decision model returned an invalid response: {exc}") from None + except DecisionEvaluationException as exc: + if not observations: + self._observe(DecisionRequestObservation(0, "error"), observations) + exc.observations = tuple(observations) + raise + + def _observe(self, observation: DecisionRequestObservation, observations: List[DecisionRequestObservation]) -> None: + observations.append(observation) + if self._execution_policy.observer is not None: + try: + self._execution_policy.observer(observation) + except Exception: # pylint: disable=broad-except + error = _failed("The decision transport observer failed.", ErrorBlame.USER_ERROR) + error.observations = tuple(observations) + raise error from None + + def _cancelled(self) -> bool: + callback = self._execution_policy.cancellation_requested + if callback is None: + return False try: - payload = await asyncio.wait_for(self._post(state, questions), timeout=timeout) - except asyncio.TimeoutError: - raise _failed("The decision model request timed out.") from None + result = callback() + except Exception: # pylint: disable=broad-except + raise _failed("The decision cancellation callback failed.", ErrorBlame.USER_ERROR) from None + if not isinstance(result, bool): + raise _failed("The decision cancellation callback must return a boolean.", ErrorBlame.USER_ERROR) + return result + + async def _await_request(self, content: bytes, observation_policy: _DecisionObservationPolicy) -> Any: + observations = observation_policy.observations + if self._cancelled(): + self._observe(DecisionRequestObservation(0, "cancelled"), observations) + raise _failed("The decision model request was cancelled.", ErrorBlame.USER_ERROR) + if self._execution_policy.cancellation_requested is None: + return await self._post(content, observation_policy) + task = asyncio.create_task(self._post(content, observation_policy)) try: - return self._validate_response(payload, questions) - except ValueError as exc: - raise _failed(f"The decision model returned an invalid response: {exc}") from None + while not task.done(): + if self._cancelled(): + self._observe(DecisionRequestObservation(observation_policy.attempt, "cancelled"), observations) + raise _failed("The decision model request was cancelled.", ErrorBlame.USER_ERROR) + await asyncio.wait({task}, timeout=0.05) + return await task + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) async def _get_token(self) -> str: if self._credential is None: - return await asyncio.to_thread(asyncio.run, self._get_default_token()) + return await asyncio.to_thread(lambda: asyncio.run(self._get_default_token())) + if inspect.iscoroutinefunction(self._credential.get_token): + return (await self._credential.get_token(_DECISION_TOKEN_SCOPE)).token token = await asyncio.to_thread(self._credential.get_token, _DECISION_TOKEN_SCOPE) if inspect.isawaitable(token): token = await token @@ -249,8 +361,9 @@ async def _get_default_token() -> str: async with AsyncAzureTokenProvider() as provider: return (await provider.get_token(_DECISION_TOKEN_SCOPE)).token - async def _post(self, state: Any, questions: Dict[str, Dict[str, Any]]) -> Any: - headers = {"x-ms-useragent": self._user_agent} + async def _post(self, content: bytes, observation_policy: _DecisionObservationPolicy) -> Any: + observations = observation_policy.observations + headers = {"x-ms-useragent": self._user_agent, "Content-Type": "application/json"} if self._api_key: headers["api-key"] = self._api_key else: @@ -261,30 +374,79 @@ async def _post(self, state: Any, questions: Dict[str, Dict[str, Any]]) -> Any: except Exception: # pylint: disable=broad-except raise _failed("Could not acquire a token for the decision model.", ErrorBlame.USER_ERROR) from None - body = {"model": self._deployment, "state": state, "questions": questions} - policies = { - "retry_policy": AsyncRetryPolicy( - retry_total=3, + policies: Dict[str, Any] = { + "retry_policy": _DecisionRetryPolicy( + retry_total=self._execution_policy.max_retries, + retry_connect=self._execution_policy.max_retries, + retry_read=self._execution_policy.max_retries, + retry_status=self._execution_policy.max_retries, retry_backoff_factor=0.8, retry_on_status_codes=_DECISION_RETRY_STATUS_CODES, ), "redirect_policy": AsyncRedirectPolicy(permit_redirects=False), } + options: Dict[str, Any] = {"retry_on_methods": ["POST"]} + if self._execution_policy.timeout_seconds is not None: + options.update( + connection_timeout=self._execution_policy.timeout_seconds, + read_timeout=self._execution_policy.timeout_seconds, + ) + + policies["custom_hook_policy"] = observation_policy try: async with get_async_http_client().with_policies(**policies) as client: - response = await client.post(self._url, json=body, headers=headers, retry_on_methods=["POST"]) + response = await client.post(self._url, content=content, headers=headers, **options) status = response.status_code + if not observations: + self._observe(self._response_observation(response, 1), observations) content = response.body() if status == 200 else b"" + except EvaluationException: + raise except Exception: # pylint: disable=broad-except + if not observations or observations[-1].outcome != "error": + self._observe(DecisionRequestObservation(observation_policy.attempt or 1, "error"), observations) raise _failed("The decision model request failed; check the endpoint and network access.") from None if status != 200: blame = ErrorBlame.USER_ERROR if status in (400, 401, 403, 404, 422) else ErrorBlame.SYSTEM_ERROR raise _failed(f"The decision model returned HTTP {status}.", blame) + if ( + self._execution_policy.max_response_bytes is not None + and len(content) > self._execution_policy.max_response_bytes + ): + raise _failed("The decision response exceeds the configured byte budget.") try: return json.loads(content) except (ValueError, TypeError): raise _failed("The decision model returned invalid JSON.") from None + def _response_observation(self, response: Any, attempt: int) -> DecisionRequestObservation: + content = response.body() + payload: Any = None + if ( + self._execution_policy.max_response_bytes is None + or len(content) <= self._execution_policy.max_response_bytes + ): + try: + payload = json.loads(content) + except (ValueError, TypeError): + pass + usage = payload.get("usage") if isinstance(payload, dict) else None + usage = usage if isinstance(usage, dict) else {} + + def token_count(name: str) -> Optional[int]: + value = usage.get(name) + return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None + + model = payload.get("model") if isinstance(payload, dict) else None + return DecisionRequestObservation( + attempt=attempt, + outcome="response", + http_status=response.status_code, + input_tokens=token_count("input_tokens"), + output_tokens=token_count("output_tokens"), + model=model if isinstance(model, str) and model.strip() else None, + ) + @staticmethod def _validate_response(payload: Any, questions: Dict[str, Dict[str, Any]]) -> Dict[str, Any]: # Only the response's structure is checked here; answer values are validated by the evaluator after it diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py index 879a366c8417..9ce6d9398f1a 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py @@ -7,15 +7,19 @@ import asyncio import json import threading +from dataclasses import FrozenInstanceError +from concurrent.futures import ThreadPoolExecutor from types import SimpleNamespace from unittest.mock import AsyncMock, Mock, patch import pytest from azure.core.pipeline.policies import AsyncRedirectPolicy, AsyncRetryPolicy from azure.core.pipeline.transport import AsyncHttpTransport, HttpResponse +from azure.core.exceptions import ServiceRequestError from azure.ai.evaluation import FluencyEvaluator from azure.ai.evaluation._evaluators._common import EvaluatorBase, ModelEvaluatorBase, PromptyEvaluatorBase +from azure.ai.evaluation._evaluators._common import DecisionExecutionPolicy, DecisionRequestObservation from azure.ai.evaluation._evaluators._common import _model_runners from azure.ai.evaluation._evaluators._common._decision_strategy import ( BinaryScoring, @@ -99,8 +103,10 @@ async def __aenter__(self): async def __aexit__(self, *_): return False - async def post(self, url, json=None, headers=None, **options): # pylint: disable=redefined-outer-name - self.calls.append({"url": url, "json": json, "headers": dict(headers), "options": options}) + async def post(self, url, content=None, headers=None, **options): + self.calls.append( + {"url": url, "json": json.loads(content), "content": content, "headers": dict(headers), "options": options} + ) if self.delay: await asyncio.sleep(self.delay) return self.response @@ -248,9 +254,14 @@ async def send(self, request, **kwargs): transport = Transport() monkeypatch.setattr(_model_runners, "get_async_http_client", lambda: AsyncHttpPipeline(transport=transport)) - result = run(DecisionModelRunner(DECISION_CONFIG)) + observations = [] + result = run( + DecisionModelRunner(DECISION_CONFIG, execution_policy=DecisionExecutionPolicy(observer=observations.append)) + ) assert transport.calls == 2 assert result["model"] == "microsoft-decision-1" + assert [receipt.attempt for receipt in observations] == [1, 2] + assert [receipt.http_status for receipt in observations] == [status, 200] def test_default_credential_token_acquisition_runs_off_event_loop(self, client): from azure.ai.evaluation._legacy._common._async_token_provider import AsyncAzureTokenProvider @@ -551,3 +562,496 @@ def test_extra_properties_are_added_with_the_decision_provenance(self): ) properties = asyncio.run(strategy.evaluate({"calls": [1, 2]}))["tool_call_accuracy_properties"] assert properties["calls"] == 2 and properties["decision_model"]["deployment"] == "Decision-1" + + def test_skipped_unusable_score_retains_known_usage_and_provenance(self): + payload = level_payload(4, status=0.1) + payload["answers"]["tool_call_accuracy"]["probabilities"] = None + result = asyncio.run(strategy_with(payload, LikertScoring(5)).evaluate({})) + assert result["tool_call_accuracy_status"] == "skipped" + assert result["tool_call_accuracy"] is None + properties = result["tool_call_accuracy_properties"] + assert (properties["prompt_tokens"], properties["completion_tokens"], properties["total_tokens"]) == ( + 120, + 3, + 123, + ) + assert properties["model"] == "microsoft-decision-1" + assert properties["decision_model"]["status_completion_probability"] == 0.1 + assert "native_answer" not in properties["decision_model"] + assert "sample_input" not in properties and "finish_reason" not in properties + + def test_semantic_failure_retains_known_usage_and_provenance(self): + payload = level_payload(4) + payload["answers"]["tool_call_accuracy"]["confidence"] = float("nan") + with pytest.raises(EvaluationException) as raised: + asyncio.run(strategy_with(payload, LikertScoring(5)).evaluate({})) + assert raised.value.properties["total_tokens"] == 123 + assert raised.value.properties["decision_model"]["deployment"] == "Decision-1" + + @pytest.mark.parametrize("status, expected", [(0.4999, "skipped"), (0.5, "completed")]) + def test_applicability_cutoff_remains_fixed(self, status, expected): + result = asyncio.run(strategy_with(level_payload(4, status=status), LikertScoring(5)).evaluate({})) + assert result["tool_call_accuracy_status"] == expected + assert result["tool_call_accuracy_properties"]["decision_model"]["status_probability_threshold"] == 0.5 + + def test_successful_properties_preserve_common_metadata_shape(self): + properties = asyncio.run(strategy_with(level_payload(4), LikertScoring(5)).evaluate({}))[ + "tool_call_accuracy_properties" + ] + assert set(properties) == { + "decision_model", + "prompt_tokens", + "completion_tokens", + "total_tokens", + "finish_reason", + "model", + "sample_input", + "sample_output", + } + assert {key: value for key, value in properties.items() if key != "decision_model"} == ( + ModelEvaluatorBase._get_token_metadata( + { + "input_token_count": 120, + "output_token_count": 3, + "total_token_count": 123, + "model_id": "microsoft-decision-1", + } + ) + ) + + +class TestDecisionExecutionPolicy: + def test_policy_and_receipts_are_immutable(self): + with pytest.raises(FrozenInstanceError): + DecisionExecutionPolicy().max_retries = 4 + with pytest.raises(FrozenInstanceError): + DecisionRequestObservation(1, "response").attempt = 2 + + @pytest.mark.parametrize( + "options", + [ + {"timeout_seconds": 0}, + {"timeout_seconds": True}, + {"timeout_seconds": float("nan")}, + {"deadline_seconds": -1}, + {"deadline_seconds": float("inf")}, + {"max_retries": True}, + {"max_retries": -1}, + {"max_retries": 1.5}, + {"max_request_bytes": 0}, + {"max_state_question_bytes": False}, + {"max_response_bytes": "100"}, + {"observer": "callback"}, + {"cancellation_requested": False}, + ], + ) + def test_invalid_policy(self, options): + with pytest.raises(ValueError): + DecisionExecutionPolicy(**options) + + def test_base_forwards_policy_outside_strict_config(self): + policy = DecisionExecutionPolicy(max_retries=0) + evaluator = ProbeModelEvaluator( + result_key="probe", model_config=DECISION_CONFIG, decision_execution_policy=policy + ) + assert evaluator._decision_model_runner._execution_policy is policy + with pytest.raises(EvaluationException, match="Unsupported"): + DecisionModelRunner({**DECISION_CONFIG, "decision_execution_policy": policy}) + with pytest.raises(EvaluationException, match="DecisionExecutionPolicy"): + DecisionModelRunner(DECISION_CONFIG, execution_policy={}) + + def test_explicit_policy_on_prompty_is_rejected_without_loading_a_flow(self): + with patch.object(_model_runners.AsyncPrompty, "load") as load: + with pytest.raises(EvaluationException, match="only supported"): + ProbeModelEvaluator( + result_key="probe", + prompty_file="probe.prompty", + model_config=CHAT_CONFIG, + decision_execution_policy=DecisionExecutionPolicy(), + ) + load.assert_not_called() + + def test_transport_timeout_and_single_retry_policy(self, client): + client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) + policy = DecisionExecutionPolicy(timeout_seconds=2, max_retries=7) + run(DecisionModelRunner(DECISION_CONFIG, execution_policy=policy)) + retry = client.policies["retry_policy"] + assert retry.total_retries == retry.connect_retries == retry.read_retries == retry.status_retries == 7 + assert client.calls[0]["options"]["connection_timeout"] == 2 + assert client.calls[0]["options"]["read_timeout"] == 2 + + @pytest.mark.parametrize("field", ["max_request_bytes", "max_state_question_bytes"]) + def test_request_budget_exact_boundary_and_no_send_on_overflow(self, client, field): + state = {"query": "\u00e9"} + if field == "max_request_bytes": + body = {"model": "Decision-1", "state": state, "questions": QUESTIONS} + size = len(json.dumps(body, ensure_ascii=False, separators=(",", ":"), allow_nan=False).encode("utf-8")) + else: + size = max( + len( + json.dumps( + {"state": state, "question": question}, + ensure_ascii=False, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + ) + for question in QUESTIONS.values() + ) + client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) + run( + DecisionModelRunner(DECISION_CONFIG, execution_policy=DecisionExecutionPolicy(**{field: size})), state=state + ) + assert b"\xc3\xa9" in client.calls[0]["content"] and b"\\u00e9" not in client.calls[0]["content"] + client.calls.clear() + with pytest.raises(EvaluationException, match="byte budget") as raised: + run( + DecisionModelRunner(DECISION_CONFIG, execution_policy=DecisionExecutionPolicy(**{field: size - 1})), + state=state, + ) + assert not client.calls + assert raised.value.observations == (DecisionRequestObservation(0, "request_rejected"),) + + def test_non_json_request_is_rejected_before_authentication_or_network(self, client): + with pytest.raises(EvaluationException, match="JSON-serializable") as raised: + run(DecisionModelRunner(DECISION_CONFIG), state={"value": float("nan")}) + assert not client.calls and raised.value.observations[0].outcome == "request_rejected" + + def test_response_budget_exact_boundary(self, client): + content = json.dumps(level_payload(4)).encode() + client.response = FakeResponse(body=content) + run( + DecisionModelRunner( + DECISION_CONFIG, execution_policy=DecisionExecutionPolicy(max_response_bytes=len(content)) + ) + ) + with pytest.raises(EvaluationException, match="byte budget"): + run( + DecisionModelRunner( + DECISION_CONFIG, execution_policy=DecisionExecutionPolicy(max_response_bytes=len(content) - 1) + ) + ) + + def test_structural_failure_preserves_usage_before_answer_validation(self, client): + payload = level_payload(4) + payload["answers"] = {} + client.response = FakeResponse(body=json.dumps(payload).encode()) + receipts = [] + with pytest.raises(EvaluationException, match="invalid response") as raised: + run( + DecisionModelRunner(DECISION_CONFIG, execution_policy=DecisionExecutionPolicy(observer=receipts.append)) + ) + assert receipts == list(raised.value.observations) + assert receipts[0].input_tokens == 120 and receipts[0].output_tokens == 3 + assert receipts[0].model == "microsoft-decision-1" + + def test_unknown_or_invalid_usage_is_not_fabricated(self, client): + client.response = FakeResponse(body=json.dumps({"usage": {"input_tokens": True, "output_tokens": 3}}).encode()) + with pytest.raises(EvaluationException) as raised: + run(DecisionModelRunner(DECISION_CONFIG)) + receipt = raised.value.observations[0] + assert receipt.input_tokens is None and receipt.output_tokens == 3 and receipt.model is None + + def test_observer_failure_is_explicit_sanitized_and_retains_receipt(self, client): + client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) + observer = Mock(side_effect=RuntimeError("observer-secret")) + with pytest.raises(EvaluationException, match="observer failed") as raised: + run(DecisionModelRunner(DECISION_CONFIG, execution_policy=DecisionExecutionPolicy(observer=observer))) + assert "observer-secret" not in str(raised.value) + assert raised.value.observations[0].input_tokens == 120 + + def test_deadline_caps_existing_evaluator_timeout(self, client): + client.delay = 1 + with pytest.raises(EvaluationException, match="timed out") as raised: + run(DecisionModelRunner(DECISION_CONFIG, execution_policy=DecisionExecutionPolicy(deadline_seconds=0.01))) + assert raised.value.observations[-1].outcome == "timeout" + + def test_evaluator_timeout_still_wins_over_longer_policy_deadline(self, client): + client.delay = 1 + with pytest.raises(EvaluationException, match="timed out"): + run( + DecisionModelRunner(DECISION_CONFIG, execution_policy=DecisionExecutionPolicy(deadline_seconds=5)), + timeout=0.01, + ) + + def test_precancellation_does_not_send(self, client): + with pytest.raises(EvaluationException, match="cancelled") as raised: + run( + DecisionModelRunner( + DECISION_CONFIG, execution_policy=DecisionExecutionPolicy(cancellation_requested=lambda: True) + ) + ) + assert not client.calls and raised.value.observations[-1].outcome == "cancelled" + + def test_inflight_cancellation_cleans_up_request(self, monkeypatch): + cleaned_up = [] + + class WaitingClient(FakeClient): + async def post(self, *args, **kwargs): + try: + await asyncio.sleep(10) + finally: + cleaned_up.append(True) + + monkeypatch.setattr(_model_runners, "get_async_http_client", WaitingClient) + + async def request(): + cancelled = False + + async def cancel(): + nonlocal cancelled + await asyncio.sleep(0.01) + cancelled = True + + cancellation = asyncio.create_task(cancel()) + try: + runner = DecisionModelRunner( + DECISION_CONFIG, execution_policy=DecisionExecutionPolicy(cancellation_requested=lambda: cancelled) + ) + with pytest.raises(EvaluationException, match="cancelled") as raised: + await runner(state={}, questions=QUESTIONS, timeout=1) + assert raised.value.observations[-1].outcome == "cancelled" + finally: + await cancellation + + asyncio.run(request()) + assert cleaned_up == [True] + + @pytest.mark.parametrize("value", [None, "yes", 1]) + def test_cancellation_callback_requires_boolean(self, client, value): + with pytest.raises(EvaluationException, match="boolean"): + run( + DecisionModelRunner( + DECISION_CONFIG, execution_policy=DecisionExecutionPolicy(cancellation_requested=lambda: value) + ) + ) + assert not client.calls + + @pytest.mark.parametrize("field", ["max_request_bytes", "max_state_question_bytes"]) + def test_tiny_positive_budget_rejects_before_send(self, client, field): + with pytest.raises(EvaluationException, match="byte budget"): + run(DecisionModelRunner(DECISION_CONFIG, execution_policy=DecisionExecutionPolicy(**{field: 1}))) + assert not client.calls + + def test_callback_failure_during_authentication_retains_error_receipt(self, client): + config = {key: value for key, value in DECISION_CONFIG.items() if key != "api_key"} + credential = Mock(get_token=Mock(side_effect=RuntimeError("credential-secret"))) + observer = Mock(side_effect=RuntimeError("observer-secret")) + with pytest.raises(EvaluationException, match="observer failed") as raised: + run(DecisionModelRunner(config, credential, execution_policy=DecisionExecutionPolicy(observer=observer))) + assert raised.value.observations == (DecisionRequestObservation(0, "error"),) + assert "secret" not in str(raised.value) + assert not client.calls + + @pytest.mark.parametrize("termination", ["timeout", "cancelled"]) + def test_termination_while_authenticating_does_not_send(self, client, termination): + config = {key: value for key, value in DECISION_CONFIG.items() if key != "api_key"} + cleaned_up = [] + + async def get_token(*_): + try: + await asyncio.sleep(10) + finally: + cleaned_up.append(True) + + credential = Mock(get_token=get_token) + + async def request(): + cancel = asyncio.Event() + loop = asyncio.get_running_loop() + handle = loop.call_later(0.01, cancel.set) + policy = DecisionExecutionPolicy( + deadline_seconds=0.02 if termination == "timeout" else 1, + cancellation_requested=cancel.is_set if termination == "cancelled" else None, + ) + try: + with pytest.raises(EvaluationException) as raised: + await DecisionModelRunner(config, credential, execution_policy=policy)( + state={}, questions=QUESTIONS, timeout=1 + ) + assert raised.value.observations == (DecisionRequestObservation(0, termination),) + finally: + handle.cancel() + + asyncio.run(request()) + assert cleaned_up == [True] and not client.calls + + @pytest.mark.parametrize("termination", ["timeout", "cancelled"]) + def test_queued_default_auth_does_not_create_an_unawaited_coroutine(self, client, termination): + config = {key: value for key, value in DECISION_CONFIG.items() if key != "api_key"} + entered = threading.Event() + release = threading.Event() + + def occupy_worker(): + entered.set() + release.wait(timeout=5) + + async def request(): + loop = asyncio.get_running_loop() + loop.set_default_executor(ThreadPoolExecutor(max_workers=1)) + occupied = loop.run_in_executor(None, occupy_worker) + cancel = asyncio.Event() + handle = None + try: + while not entered.is_set(): + await asyncio.sleep(0) + handle = loop.call_later(0.01, cancel.set) + policy = DecisionExecutionPolicy( + deadline_seconds=0.03 if termination == "timeout" else 1, + cancellation_requested=cancel.is_set if termination == "cancelled" else None, + ) + with patch.object(DecisionModelRunner, "_get_default_token", new_callable=AsyncMock) as get_token: + with pytest.raises(EvaluationException) as raised: + await DecisionModelRunner(config, execution_policy=policy)( + state={}, questions=QUESTIONS, timeout=1 + ) + assert raised.value.observations == (DecisionRequestObservation(0, termination),) + get_token.assert_not_called() + finally: + if handle is not None: + handle.cancel() + release.set() + await occupied + + asyncio.run(request()) + assert not client.calls + + @pytest.mark.parametrize("termination", ["timeout", "cancelled"]) + @pytest.mark.parametrize("waiting", ["send", "retry_backoff"]) + def test_real_pipeline_termination_retains_actual_attempt(self, monkeypatch, termination, waiting): + from azure.ai.evaluation._http_utils import AsyncHttpPipeline + + cancelled = False + cleaned_up = [] + + class Response(HttpResponse): + def __init__(self, request): + super().__init__(request, None) + self.status_code = 429 + self.headers = {"Retry-After": "10"} + + def body(self): + return json.dumps(level_payload(4)).encode() + + class Transport(AsyncHttpTransport): + def __init__(self): + self.calls = 0 + + async def open(self): + pass + + async def close(self): + pass + + async def __aexit__(self, *_): + await self.close() + + async def send(self, request, **kwargs): + self.calls += 1 + if waiting == "retry_backoff": + return Response(request) + try: + await asyncio.sleep(10) + finally: + cleaned_up.append(True) + + transport = Transport() + monkeypatch.setattr(_model_runners, "get_async_http_client", lambda: AsyncHttpPipeline(transport=transport)) + + async def request(): + nonlocal cancelled + loop = asyncio.get_running_loop() + + def cancel(): + nonlocal cancelled + cancelled = True + + handle = loop.call_later(0.01, cancel) + policy = DecisionExecutionPolicy( + deadline_seconds=0.08 if termination == "timeout" else 1, + cancellation_requested=(lambda: cancelled) if termination == "cancelled" else None, + ) + try: + with pytest.raises(EvaluationException) as raised: + await DecisionModelRunner(DECISION_CONFIG, execution_policy=policy)( + state={}, questions=QUESTIONS, timeout=1 + ) + receipts = raised.value.observations + assert receipts[-1] == DecisionRequestObservation(1, termination) + if waiting == "retry_backoff": + assert receipts[0].http_status == 429 and receipts[0].input_tokens == 120 + finally: + handle.cancel() + + asyncio.run(request()) + assert transport.calls == 1 + assert cleaned_up == ([True] if waiting == "send" else []) + + @pytest.mark.parametrize("observer_fails", [False, True]) + def test_real_pipeline_transport_errors_retain_each_attempt(self, monkeypatch, observer_fails): + from azure.ai.evaluation._http_utils import AsyncHttpPipeline + + class Transport(AsyncHttpTransport): + def __init__(self): + self.calls = 0 + + async def open(self): + pass + + async def close(self): + pass + + async def __aexit__(self, *_): + await self.close() + + async def send(self, request, **kwargs): + self.calls += 1 + raise ServiceRequestError("transport-secret") + + transport = Transport() + monkeypatch.setattr(_model_runners, "get_async_http_client", lambda: AsyncHttpPipeline(transport=transport)) + observer = Mock(side_effect=RuntimeError("observer-secret")) if observer_fails else Mock() + policy = DecisionExecutionPolicy(max_retries=1, observer=observer) + with pytest.raises(EvaluationException) as raised: + run(DecisionModelRunner(DECISION_CONFIG, execution_policy=policy)) + expected = 1 if observer_fails else 2 + assert transport.calls == expected + assert raised.value.observations == tuple( + DecisionRequestObservation(i, "error") for i in range(1, expected + 1) + ) + assert observer.call_count == expected + assert "secret" not in str(raised.value) + + def test_real_pipeline_observer_failure_preserves_response_receipt(self, monkeypatch): + from azure.ai.evaluation._http_utils import AsyncHttpPipeline + + class Response(HttpResponse): + def __init__(self, request): + super().__init__(request, None) + self.status_code = 200 + self.headers = {} + + def body(self): + return json.dumps(level_payload(4)).encode() + + class Transport(AsyncHttpTransport): + async def open(self): + pass + + async def close(self): + pass + + async def __aexit__(self, *_): + await self.close() + + async def send(self, request, **kwargs): + return Response(request) + + monkeypatch.setattr(_model_runners, "get_async_http_client", lambda: AsyncHttpPipeline(transport=Transport())) + observer = Mock(side_effect=RuntimeError("observer-secret")) + with pytest.raises(EvaluationException, match="observer failed") as raised: + run(DecisionModelRunner(DECISION_CONFIG, execution_policy=DecisionExecutionPolicy(observer=observer))) + assert raised.value.observations == ( + DecisionRequestObservation(1, "response", 200, 120, 3, "microsoft-decision-1"), + ) + assert observer.call_count == 1 and "secret" not in str(raised.value) From ee6719c41d6350e0f72a0fa4511bb1feae6b30a4 Mon Sep 17 00:00:00 2001 From: salma-elshafey Date: Tue, 13 Oct 2026 13:03:07 +0300 Subject: [PATCH 05/24] Retain native Decision status metadata only after validation Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e564c5c8-480e-48e7-a9e0-44450f774b5e --- .../_evaluators/_common/_decision_strategy.py | 2 +- .../unittests/test_decision_model_support.py | 40 ++++++++++++++++++- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py index 48412071fe02..04517c6169cc 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py @@ -165,7 +165,6 @@ async def evaluate(self, eval_input: Dict[str, Any]) -> Dict[str, Any]: "model": response["model"], "rubric_hash": _digest(self.questions[key]), "status_rubric_hash": _digest(self.questions["status"]), - "native_status_answer": response["answers"]["status"], "status_probability_threshold": STATUS_CUTOFF, "reason_type": "templated", } @@ -181,6 +180,7 @@ async def evaluate(self, eval_input: Dict[str, Any]) -> Dict[str, Any]: properties["decision_model"]["status_completion_probability"] = decision_probability( response, "status", self.error_target ) + properties["decision_model"]["native_status_answer"] = response["answers"]["status"] if skip_reason: result = not_applicable_result(key, skip_reason, self.threshold) result[f"{key}_properties"] = properties diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py index 9ce6d9398f1a..a614a259b51f 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py @@ -5,6 +5,7 @@ """Offline tests for shared decision-model runners, base selection, and evaluation strategy.""" import asyncio +import hashlib import json import threading from dataclasses import FrozenInstanceError @@ -588,11 +589,48 @@ def test_semantic_failure_retains_known_usage_and_provenance(self): assert raised.value.properties["total_tokens"] == 123 assert raised.value.properties["decision_model"]["deployment"] == "Decision-1" + @pytest.mark.parametrize("status", [float("nan"), float("inf"), float("-inf"), "invalid", None, True, -0.1, 1.1]) + def test_invalid_status_error_properties_are_strict_json_with_known_metadata(self, client, status): + client.response = FakeResponse(body=json.dumps(level_payload(4, status=status)).encode()) + strategy = DecisionEvaluationStrategy( + DecisionModelRunner(DECISION_CONFIG), + result_key="tool_call_accuracy", + error_target=ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR, + questions=QUESTIONS, + scoring=LikertScoring(5), + threshold=3, + timeout=5, + ) + with pytest.raises(EvaluationException) as raised: + asyncio.run(strategy.evaluate({})) + properties = raised.value.properties + assert json.loads(json.dumps(properties, allow_nan=False)) == properties + assert (properties["prompt_tokens"], properties["completion_tokens"], properties["total_tokens"]) == ( + 120, + 3, + 123, + ) + assert properties["model"] == "microsoft-decision-1" + provenance = properties["decision_model"] + assert provenance["deployment"] == "Decision-1" + assert provenance["model"] == "microsoft-decision-1" + for field, question_id in (("rubric_hash", "tool_call_accuracy"), ("status_rubric_hash", "status")): + assert ( + provenance[field] + == hashlib.sha256(json.dumps(QUESTIONS[question_id], sort_keys=True).encode("utf-8")).hexdigest() + ) + assert "native_status_answer" not in provenance + assert "status_completion_probability" not in provenance + @pytest.mark.parametrize("status, expected", [(0.4999, "skipped"), (0.5, "completed")]) def test_applicability_cutoff_remains_fixed(self, status, expected): - result = asyncio.run(strategy_with(level_payload(4, status=status), LikertScoring(5)).evaluate({})) + payload = level_payload(4, status=status) + result = asyncio.run(strategy_with(payload, LikertScoring(5)).evaluate({})) assert result["tool_call_accuracy_status"] == expected assert result["tool_call_accuracy_properties"]["decision_model"]["status_probability_threshold"] == 0.5 + assert result["tool_call_accuracy_properties"]["decision_model"]["native_status_answer"] == ( + payload["answers"]["status"] + ) def test_successful_properties_preserve_common_metadata_shape(self): properties = asyncio.run(strategy_with(level_payload(4), LikertScoring(5)).evaluate({}))[ From 11f303e598463678ef48afc289116734e10c0b08 Mon Sep 17 00:00:00 2001 From: salma-elshafey Date: Tue, 13 Oct 2026 13:45:47 +0300 Subject: [PATCH 06/24] Preserve Decision failure metadata in batch evaluation rows Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e564c5c8-480e-48e7-a9e0-44450f774b5e --- .../_batch_run/_run_submitter_client.py | 62 +++- .../ai/evaluation/_evaluate/_evaluate.py | 18 + .../test_decision_batch_persistence.py | 311 ++++++++++++++++++ 3 files changed, 390 insertions(+), 1 deletion(-) create mode 100644 sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py index 6ec1a67568b6..edd1493c0301 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py @@ -7,6 +7,7 @@ import pandas as pd import sys import itertools +import math from collections import defaultdict from concurrent.futures import Future from os import PathLike @@ -21,6 +22,7 @@ from ..._legacy._common._thread_pool_executor_with_context import ThreadPoolExecutorWithContext from ..._evaluate._utils import _has_aggregator from ..._constants import Prefixes, PF_BATCH_TIMEOUT_SEC +from ..._evaluators._common._decision_execution import DecisionEvaluationException from .._utils import get_int_env_var as get_int @@ -28,6 +30,58 @@ MISSING_VALUE: Final[int] = sys.maxsize +def _decision_error_metadata(exception: DecisionEvaluationException) -> Dict[str, Any]: + """Keep only measured usage, validated provenance, and transport receipts for failed Decision rows.""" + properties: Dict[str, Any] = {} + semantic_properties = exception.properties or {} + for key in ("prompt_tokens", "completion_tokens", "total_tokens"): + value = semantic_properties.get(key) + if isinstance(value, int) and not isinstance(value, bool) and value >= 0: + properties[key] = value + model = semantic_properties.get("model") + if isinstance(model, str) and model.strip(): + properties["model"] = model + native = semantic_properties.get("decision_model") + if isinstance(native, dict): + provenance = { + key: native[key] + for key in ("deployment", "model", "rubric_hash", "status_rubric_hash", "reason_type") + if isinstance(native.get(key), str) + } + for key in ("status_completion_probability", "status_probability_threshold"): + value = native.get(key) + if ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + and 0 <= value <= 1 + ): + provenance[key] = value + if provenance: + properties["decision_model"] = provenance + if exception.observations: + receipts = [] + for observation in exception.observations: + receipt = { + "attempt": observation.attempt, + "outcome": observation.outcome, + "http_status": ( + observation.http_status + if isinstance(observation.http_status, int) and not isinstance(observation.http_status, bool) + else None + ), + "model": ( + observation.model if isinstance(observation.model, str) and observation.model.strip() else None + ), + } + for key in ("input_tokens", "output_tokens"): + value = getattr(observation, key) + receipt[key] = value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None + receipts.append(receipt) + properties["decision_request_observations"] = receipts + return {"code": exception.category.value, "blame": exception.blame.value, "properties": properties} + + class RunSubmitterClient: def __init__(self, *, raise_on_errors: bool = False, config: Optional[BatchEngineConfig] = None) -> None: if config: @@ -154,12 +208,15 @@ def get_run_summary(self, client_run: BatchClientRun) -> Dict[str, Any]: # Collect per-line error messages for failed lines per_line_errors: Dict[int, str] = {} + per_line_error_metadata: Dict[int, Dict[str, Any]] = {} if run.result and run.result.details: for detail in run.result.details: if detail.error and detail.error.exception: per_line_errors[detail.index] = str(detail.error.exception) + if isinstance(detail.error.exception, DecisionEvaluationException): + per_line_error_metadata[detail.index] = _decision_error_metadata(detail.error.exception) - return { + summary = { "status": run.status.value, "duration": str(run.duration), "completed_lines": total_lines - failed_lines, @@ -177,6 +234,9 @@ def get_run_summary(self, client_run: BatchClientRun) -> Dict[str, Any]: else None ), } + if per_line_error_metadata: + summary["per_line_error_metadata"] = per_line_error_metadata + return summary @staticmethod def _get_run(run: BatchClientRun) -> Run: diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py index 70e9bcccd701..fbf859e0b589 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py @@ -3301,6 +3301,8 @@ def _add_error_summaries( for criteria that failed during evaluation, ensuring proper error reporting. When per-line errors are available, only the error for the current row is used; rows that succeeded are not stamped with another row's error. + Decision-only per-line metadata preserves known usage/provenance in ``properties`` and the + existing ``sample`` schema, without treating failed attempts as completed scoring results. :param run_output_results: List to append error result objects to :type run_output_results: List[Dict[str, Any]] @@ -3357,6 +3359,7 @@ def _add_error_summaries( # Use per-line error if available, otherwise fall back to batch-level error per_line_errors = criteria_summary.get("per_line_errors", {}) per_line_error_msg = per_line_errors.get(row_idx, None) + decision_error_metadata = criteria_summary.get("per_line_error_metadata", {}).get(row_idx) if per_line_error_msg is not None: # This row has a specific error @@ -3364,6 +3367,9 @@ def _add_error_summaries( "code": criteria_summary.get("error_code"), "message": per_line_error_msg, } + if decision_error_metadata is not None: + error_info["code"] = decision_error_metadata["code"] + error_info["blame"] = decision_error_metadata["blame"] elif per_line_errors: # Per-line errors exist but not for this row — this row succeeded, skip error stamping continue @@ -3395,6 +3401,18 @@ def _add_error_summaries( "status": "error", "sample": sample, } + if decision_error_metadata is not None: + properties = decision_error_metadata["properties"] + error_result["properties"] = properties + usage = { + key: properties[key] + for key in ("prompt_tokens", "completion_tokens", "total_tokens") + if key in properties + } + if usage: + sample["usage"] = usage + if "model" in properties: + sample["model"] = properties["model"] run_output_results.append(error_result) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py new file mode 100644 index 000000000000..dd1f8ecad78d --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py @@ -0,0 +1,311 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- + +"""Decision failure metadata through the actual batch summary and row conversion.""" + +import asyncio +import json +import logging + +import pandas as pd +import pytest +from azure.core.pipeline.transport import AsyncHttpTransport, HttpResponse + +from azure.ai.evaluation import F1ScoreEvaluator +from azure.ai.evaluation._evaluate._batch_run import RunSubmitterClient +from azure.ai.evaluation._evaluate._evaluate import _convert_single_row_to_aoai_format +from azure.ai.evaluation._evaluators._common import DecisionExecutionPolicy, DecisionRequestObservation +from azure.ai.evaluation._evaluators._common._decision_execution import DecisionEvaluationException +from azure.ai.evaluation._evaluators._common._decision_strategy import BinaryScoring, DecisionEvaluationStrategy +from azure.ai.evaluation._evaluators._common._model_runners import DecisionModelRunner +from azure.ai.evaluation._evaluators._common import _model_runners +from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException +from azure.ai.evaluation._http_utils import AsyncHttpPipeline + +pytestmark = pytest.mark.unittest + +CONFIG = { + "type": "decision", + "azure_endpoint": "https://example.services.ai.azure.com", + "azure_deployment": "Decision-1", + "api_key": "test-key-not-for-persistence", +} +QUESTIONS = { + "status": {"type": "noul", "instructions": "Applicable?"}, + "task_completion": {"type": "noul", "instructions": "Completed?"}, +} +CRITERIA = { + "judge": {"metrics": ["task_completion"], "type": "azure_ai_evaluator"}, + "f1": {"metrics": ["f1_score"], "type": "azure_ai_evaluator"}, +} + + +def payload(status=0.9): + return { + "model": "decision-native", + "answers": {"status": {"type": "noul", "noul": status}, "task_completion": {"type": "noul", "noul": 0.9}}, + "usage": {"input_tokens": 120, "output_tokens": 3}, + } + + +@pytest.fixture +def batch_client(): + client = RunSubmitterClient() + try: + yield client + finally: + client._thread_pool.shutdown(wait=True) + + +def convert_batch(client, judge, tmp_path, rows=1): + data = pd.DataFrame([{"query": "synthetic", "response": "synthetic", "ground_truth": "synthetic"}] * rows) + judge_run = client.run(judge, data, evaluator_name="judge", column_mapping={"query": "${data.query}"}) + f1_run = client.run( + F1ScoreEvaluator(), + data, + evaluator_name="f1", + column_mapping={"response": "${data.response}", "ground_truth": "${data.ground_truth}"}, + ) + judge_details = client.get_details(judge_run, all_results=True) + f1_details = client.get_details(f1_run, all_results=True) + summaries = {"judge": client.get_run_summary(judge_run), "f1": client.get_run_summary(f1_run)} + converted = [] + for index in range(rows): + row = {} + for name, details in (("judge", judge_details), ("f1", f1_details)): + for key, value in details.to_dict(orient="records")[index].items(): + row[key.replace("outputs.", f"outputs.{name}.", 1)] = value + converted.append( + _convert_single_row_to_aoai_format( + row, index, None, None, 0, CRITERIA, summaries, logging.getLogger("test") + ) + ) + path = tmp_path / "persisted.json" + path.write_text(json.dumps(converted, allow_nan=False), encoding="utf-8") + persisted = json.loads(path.read_text(encoding="utf-8")) + for row in persisted: + f1 = next(result for result in row["results"] if result["name"] == "f1") + assert f1["score"] == 1 and f1["status"] == "completed" + assert "error" not in (f1.get("sample") or {}) + return summaries, persisted + + +def strategy(runner): + return DecisionEvaluationStrategy( + runner, + result_key="task_completion", + error_target=ErrorTarget.EVALUATE, + questions=QUESTIONS, + scoring=BinaryScoring(0.75), + threshold=1, + timeout=2, + ) + + +class TestDecisionBatchPersistence: + @pytest.mark.parametrize("status", [float("nan"), float("inf"), "bad"]) + def test_semantic_failure_retains_measured_usage_and_hashes(self, batch_client, tmp_path, status): + class Runner: + deployment = "Decision-1" + + async def __call__(self, **kwargs): + return payload(status) + + async def judge(query): + return await strategy(Runner()).evaluate({"query": query}) + + summary, rows = convert_batch(batch_client, judge, tmp_path) + result = next(item for item in rows[0]["results"] if item["name"] == "judge") + assert result["status"] == "error" and result["score"] is None and result["passed"] is None + assert result["sample"]["error"]["blame"] == "SystemError" + assert result["sample"]["error"]["code"] == "FAILED_EXECUTION" + assert result["sample"]["usage"] == {"prompt_tokens": 120, "completion_tokens": 3, "total_tokens": 123} + assert result["sample"]["model"] == "decision-native" + assert result["properties"]["total_tokens"] == 123 + assert len(result["properties"]["decision_model"]["rubric_hash"]) == 64 + assert "native_status_answer" not in result["properties"]["decision_model"] + assert summary["judge"]["failed_lines"] == 1 + + @pytest.mark.parametrize("status, expected", [(0.9, "completed"), (0.1, "skipped")]) + def test_success_and_skip_are_not_stamped_as_errors(self, batch_client, tmp_path, status, expected): + class Runner: + deployment = "Decision-1" + + async def __call__(self, **kwargs): + return payload(status) + + async def judge(query): + return await strategy(Runner()).evaluate({"query": query}) + + summary, rows = convert_batch(batch_client, judge, tmp_path) + result = next(item for item in rows[0]["results"] if item["name"] == "judge") + assert result["status"] == expected + assert "error" not in (result.get("sample") or {}) + assert "per_line_error_metadata" not in summary["judge"] + + @pytest.mark.parametrize( + "case", ["malformed_shape", "invalid_json", "partial_usage", "precancel", "inflight_cancel", "billed_cancel"] + ) + def test_real_pipeline_failure_receipts_survive_batch_conversion(self, batch_client, tmp_path, monkeypatch, case): + cancelled = False + + class Response(HttpResponse): + def __init__(self, request): + super().__init__(request, None) + self.status_code = 429 if case == "billed_cancel" else 200 + self.headers = {"Retry-After": "10"} if case == "billed_cancel" else {} + + def body(self): + if case == "invalid_json": + return b"not-json" + return json.dumps( + { + "model": "decision-native", + "answers": {}, + "usage": {"input_tokens": True if case == "partial_usage" else 12, "output_tokens": 4}, + } + ).encode() + + class Transport(AsyncHttpTransport): + async def open(self): + pass + + async def close(self): + pass + + async def __aexit__(self, *_): + await self.close() + + async def send(self, request, **kwargs): + if case == "inflight_cancel": + await asyncio.sleep(10) + return Response(request) + + monkeypatch.setattr(_model_runners, "get_async_http_client", lambda: AsyncHttpPipeline(transport=Transport())) + + def observer(receipt): + nonlocal cancelled + if case == "billed_cancel": + cancelled = True + + async def judge(query): + nonlocal cancelled + cancelled = case == "precancel" + handle = None + if case == "inflight_cancel": + + def cancel(): + nonlocal cancelled + cancelled = True + + handle = asyncio.get_running_loop().call_later(0.02, cancel) + try: + runner = DecisionModelRunner( + CONFIG, + execution_policy=DecisionExecutionPolicy( + max_retries=1, + cancellation_requested=lambda: cancelled, + observer=observer, + ), + ) + return await strategy(runner).evaluate({"query": query}) + finally: + if handle is not None: + handle.cancel() + + _, rows = convert_batch(batch_client, judge, tmp_path) + result = next(item for item in rows[0]["results"] if item["name"] == "judge") + assert result["status"] == "error" and result["score"] is None and result["passed"] is None + receipts = result["properties"]["decision_request_observations"] + assert all( + set(receipt) == {"attempt", "outcome", "http_status", "input_tokens", "output_tokens", "model"} + for receipt in receipts + ) + assert "usage" not in result["sample"] and "model" not in result["sample"] + if case in ("malformed_shape", "partial_usage", "billed_cancel"): + assert receipts[0]["input_tokens"] == (None if case == "partial_usage" else 12) + assert receipts[0]["output_tokens"] == 4 and receipts[0]["model"] == "decision-native" + else: + assert receipts[0]["input_tokens"] is None and receipts[0]["output_tokens"] is None + if "cancel" in case: + assert result["sample"]["error"]["blame"] == "UserError" + assert receipts[-1]["outcome"] == "cancelled" + assert receipts[-1]["attempt"] == (0 if case == "precancel" else 1) + assert "test-key-not-for-persistence" not in json.dumps(result) + assert "synthetic" not in json.dumps(result["properties"]) + + def test_per_line_classification_and_metadata_do_not_bleed(self, batch_client, tmp_path): + calls = 0 + + async def judge(query): + nonlocal calls + index = calls + calls += 1 + if index == 0: + raise DecisionEvaluationException( + "Cancelled.", + category=ErrorCategory.FAILED_EXECUTION, + blame=ErrorBlame.USER_ERROR, + observations=(DecisionRequestObservation(0, "cancelled"),), + ) + if index == 1: + raise DecisionEvaluationException( + "Invalid.", + category=ErrorCategory.INVALID_VALUE, + blame=ErrorBlame.SYSTEM_ERROR, + properties={ + "prompt_tokens": 7, + "completion_tokens": True, + "total_tokens": float("nan"), + "evidence": "secret", + "model": "decision-native", + "decision_model": {"rubric_hash": "known", "native_answer": {"noul": float("nan")}}, + }, + ) + return {"task_completion": 1, "task_completion_score": 1, "task_completion_passed": True} + + _, rows = convert_batch(batch_client, judge, tmp_path, rows=3) + first, second, third = [next(item for item in row["results"] if item["name"] == "judge") for row in rows] + assert first["sample"]["error"]["blame"] == "UserError" + assert first["sample"]["error"]["code"] == "FAILED_EXECUTION" + assert second["sample"]["error"]["blame"] == "SystemError" + assert second["sample"]["error"]["code"] == "INVALID VALUE" + assert second["sample"]["usage"] == {"prompt_tokens": 7} + assert second["properties"] == { + "prompt_tokens": 7, + "model": "decision-native", + "decision_model": {"rubric_hash": "known"}, + } + assert third["status"] == "completed" and "error" not in (third.get("sample") or {}) + + def test_other_exceptions_preserve_summary_and_error_row_shape(self, batch_client, tmp_path): + async def judge(query): + raise EvaluationException( + "Ordinary error.", category=ErrorCategory.INVALID_VALUE, blame=ErrorBlame.USER_ERROR + ) + + summaries, rows = convert_batch(batch_client, judge, tmp_path) + assert set(summaries["judge"]) == { + "status", + "duration", + "completed_lines", + "failed_lines", + "log_path", + "per_line_errors", + "error_message", + "error_code", + } + result = next(item for item in rows[0]["results"] if item["name"] == "judge") + assert result == { + "type": "azure_ai_evaluator", + "name": "judge", + "metric": "task_completion", + "score": None, + "label": None, + "reason": None, + "threshold": None, + "passed": None, + "status": "error", + "sample": {"error": {"code": summaries["judge"]["error_code"], "message": "(UserError) Ordinary error."}}, + } From 18b49f6551a804d13d38ed010f9b2ba38a94a821 Mon Sep 17 00:00:00 2001 From: salma-elshafey Date: Tue, 13 Oct 2026 15:33:19 +0300 Subject: [PATCH 07/24] Translate explicit read timeouts for the default async requests transport Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e564c5c8-480e-48e7-a9e0-44450f774b5e --- .../azure/ai/evaluation/_http_utils.py | 16 +- .../test_decision_default_transport.py | 156 ++++++++++++++++++ 2 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_default_transport.py diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_http_utils.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_http_utils.py index 0b0448f5a903..163f73a8f426 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_http_utils.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_http_utils.py @@ -249,6 +249,20 @@ def __enter__(self) -> Self: return cast(Self, super().__enter__()) +class _AsyncioRequestsTransportWithReadTimeout(AsyncioRequestsTransport): + """Adapt explicit read timeouts for the requests-backed default transport.""" + + async def send(self, request, **kwargs): + # Translate after RetryPolicy has bounded the numeric connection timeout. + # AsyncioRequestsTransport otherwise forwards read_timeout to requests, which rejects it. + if "read_timeout" in kwargs: + connection_timeout = kwargs.pop("connection_timeout", self.connection_config.timeout) + if isinstance(connection_timeout, tuple): + raise ValueError("Cannot set tuple connection_timeout and read_timeout together") + kwargs["connection_timeout"] = (connection_timeout, kwargs.pop("read_timeout")) + return await super().send(request, **kwargs) + + class AsyncHttpPipeline(AsyncPipeline): """A *very* thin wrapper over azure.core.pipeline.AsyncPipeline that facilitates sending miscellaneous http requests by adding: @@ -323,7 +337,7 @@ def __init__( # AsyncioRequestsTransport normally should not be imported outside of azure.core, since transports # are meant to be user configurable. # AsyncioRequestsTransport is only used in this file as the default transport when not user specified. - transport=transport or AsyncioRequestsTransport(**kwargs), + transport=transport or _AsyncioRequestsTransportWithReadTimeout(**kwargs), policies=[ config.headers_policy, config.user_agent_policy, diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_default_transport.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_default_transport.py new file mode 100644 index 000000000000..f26088f92f90 --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_default_transport.py @@ -0,0 +1,156 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- + +"""Actual localhost HTTP coverage for the default requests-backed async transport.""" + +import asyncio +import json +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import pytest +import requests +from urllib3.util import connection + +from azure.ai.evaluation._evaluators._common import DecisionExecutionPolicy +from azure.ai.evaluation._evaluators._common._model_runners import DecisionModelRunner +from azure.ai.evaluation._exceptions import EvaluationException +from azure.ai.evaluation._http_utils import get_async_http_client +from azure.core.pipeline.policies import AsyncRetryPolicy + +pytestmark = pytest.mark.unittest +QUESTIONS = {"status": {"type": "noul", "instructions": "Applicable?"}} + + +@pytest.fixture +def localhost(): + calls = [] + delay = [0] + statuses = [200] + payload = { + "model": "native-decision", + "answers": {"status": {"type": "noul", "noul": 0.9}}, + "usage": {"input_tokens": 12, "output_tokens": 4}, + } + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *_): + pass + + def do_POST(self): + body = self.rfile.read(int(self.headers["Content-Length"])) + calls.append({"path": self.path, "body": body, "content_type": self.headers.get("Content-Type")}) + time.sleep(delay[0]) + content = json.dumps(payload).encode() + self.send_response(statuses.pop(0) if len(statuses) > 1 else statuses[0]) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(content))) + self.end_headers() + try: + self.wfile.write(content) + except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}", calls, delay, statuses + finally: + server.shutdown() + server.server_close() + thread.join() + + +def invoke(endpoint, policy, timeout=3): + runner = DecisionModelRunner( + {"type": "decision", "azure_endpoint": endpoint, "azure_deployment": "Decision-1", "api_key": "local-test"}, + execution_policy=policy, + ) + return asyncio.run(runner(state={"query": "é"}, questions=QUESTIONS, timeout=timeout)) + + +def test_default_transport_sends_policy_request_and_retries(localhost, monkeypatch): + endpoint, calls, _, statuses = localhost + statuses[:] = [429, 200] + observed_timeouts = [] + socket_connect_timeouts = [] + original = requests.Session.request + original_connect = connection.create_connection + + def record_connect(address, timeout=None, *args, **kwargs): + socket_connect_timeouts.append(timeout) + return original_connect(address, timeout, *args, **kwargs) + + def record(session, *args, **kwargs): + observed_timeouts.append(kwargs["timeout"]) + assert "read_timeout" not in kwargs and "connection_timeout" not in kwargs + return original(session, *args, **kwargs) + + monkeypatch.setattr(requests.Session, "request", record) + monkeypatch.setattr(connection, "create_connection", record_connect) + receipts = [] + result = invoke(endpoint, DecisionExecutionPolicy(timeout_seconds=0.5, max_retries=1, observer=receipts.append)) + assert result["usage"] == {"input_tokens": 12, "output_tokens": 4} + assert observed_timeouts == [(0.5, 0.5), (0.5, 0.5)] + assert socket_connect_timeouts == [0.5, 0.5] + assert [receipt.http_status for receipt in receipts] == [429, 200] + assert [receipt.attempt for receipt in receipts] == [1, 2] + assert len(calls) == 2 and calls[0]["body"] == calls[1]["body"] + assert calls[0]["path"] == "/providers/microsoft/v1/systemone" + assert calls[0]["content_type"] == "application/json" + assert json.loads(calls[0]["body"]) == {"model": "Decision-1", "state": {"query": "é"}, "questions": QUESTIONS} + assert b"\xc3\xa9" in calls[0]["body"] + + +def test_default_transport_read_timeout_is_enforced(localhost): + endpoint, calls, delay, _ = localhost + delay[0] = 0.5 + started = time.monotonic() + with pytest.raises(EvaluationException) as raised: + invoke(endpoint, DecisionExecutionPolicy(timeout_seconds=0.05, max_retries=0)) + assert time.monotonic() - started < 0.4 + assert len(calls) == 1 + assert raised.value.observations[0].attempt == 1 + assert raised.value.observations[0].outcome == "error" + + +def test_default_transport_numeric_retry_deadline_and_separate_timeout_bounds(localhost, monkeypatch): + endpoint, _, _, _ = localhost + original = requests.Session.request + observed = [] + + def record(session, *args, **kwargs): + observed.append(kwargs["timeout"]) + return original(session, *args, **kwargs) + + monkeypatch.setattr(requests.Session, "request", record) + + async def request(): + async with get_async_http_client(retry_policy=AsyncRetryPolicy(retry_total=0)) as client: + return await client.post(endpoint, content=b"{}", connection_timeout=2, read_timeout=0.3, timeout=0.8) + + assert asyncio.run(request()).status_code == 200 + assert 0 < observed[0][0] <= 0.8 + assert observed[0][1] == 0.3 + + +def test_default_transport_without_read_override_preserves_scalar_timeout(localhost, monkeypatch): + endpoint, _, _, _ = localhost + original = requests.Session.request + observed = [] + + def record(session, *args, **kwargs): + observed.append(kwargs["timeout"]) + return original(session, *args, **kwargs) + + monkeypatch.setattr(requests.Session, "request", record) + + async def request(): + async with get_async_http_client() as client: + return await client.post(endpoint, content=b"{}", connection_timeout=0.7) + + assert asyncio.run(request()).status_code == 200 + assert observed == [0.7] From 8afa360679474bb2eb3016bbf7416af6002f5a3c Mon Sep 17 00:00:00 2001 From: salma-elshafey Date: Tue, 13 Oct 2026 17:11:57 +0300 Subject: [PATCH 08/24] Simplify Decision execution controls and map measured usage into standard rows Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e564c5c8-480e-48e7-a9e0-44450f774b5e --- .../azure-ai-evaluation/CHANGELOG.md | 1 + .../_batch_run/_run_submitter_client.py | 38 +-- .../_evaluators/_common/__init__.py | 3 - .../_evaluators/_common/_base_model_eval.py | 64 ++++- .../_common/_decision_execution.py | 81 ------ .../_evaluators/_common/_decision_strategy.py | 13 +- .../_evaluators/_common/_model_runners.py | 216 +++++++------- .../azure/ai/evaluation/_exceptions.py | 6 +- .../azure/ai/evaluation/_http_utils.py | 16 +- .../test_decision_batch_persistence.py | 83 ++++-- .../test_decision_default_transport.py | 53 ++-- .../unittests/test_decision_model_support.py | 272 +++++++++--------- 12 files changed, 404 insertions(+), 442 deletions(-) delete mode 100644 sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_execution.py diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index e0f44299e9aa..f0b8ccd753ed 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -5,6 +5,7 @@ ### Features Added - Added shared model runners and evaluation strategies for asset evaluators using decision models in Microsoft Foundry. +- Decision evaluator results expose measured token usage and model fields in standard evaluation rows, including skipped evaluations, while retaining known metadata on failures. ### Breaking Changes diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py index edd1493c0301..34cbd602475c 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py @@ -22,7 +22,7 @@ from ..._legacy._common._thread_pool_executor_with_context import ThreadPoolExecutorWithContext from ..._evaluate._utils import _has_aggregator from ..._constants import Prefixes, PF_BATCH_TIMEOUT_SEC -from ..._evaluators._common._decision_execution import DecisionEvaluationException +from ..._exceptions import EvaluationException from .._utils import get_int_env_var as get_int @@ -30,8 +30,8 @@ MISSING_VALUE: Final[int] = sys.maxsize -def _decision_error_metadata(exception: DecisionEvaluationException) -> Dict[str, Any]: - """Keep only measured usage, validated provenance, and transport receipts for failed Decision rows.""" +def _decision_error_metadata(exception: EvaluationException) -> Dict[str, Any]: + """Keep only known final-response usage and validated provenance for failed Decision rows.""" properties: Dict[str, Any] = {} semantic_properties = exception.properties or {} for key in ("prompt_tokens", "completion_tokens", "total_tokens"): @@ -57,28 +57,7 @@ def _decision_error_metadata(exception: DecisionEvaluationException) -> Dict[str and 0 <= value <= 1 ): provenance[key] = value - if provenance: - properties["decision_model"] = provenance - if exception.observations: - receipts = [] - for observation in exception.observations: - receipt = { - "attempt": observation.attempt, - "outcome": observation.outcome, - "http_status": ( - observation.http_status - if isinstance(observation.http_status, int) and not isinstance(observation.http_status, bool) - else None - ), - "model": ( - observation.model if isinstance(observation.model, str) and observation.model.strip() else None - ), - } - for key in ("input_tokens", "output_tokens"): - value = getattr(observation, key) - receipt[key] = value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None - receipts.append(receipt) - properties["decision_request_observations"] = receipts + properties["decision_model"] = provenance return {"code": exception.category.value, "blame": exception.blame.value, "properties": properties} @@ -213,8 +192,13 @@ def get_run_summary(self, client_run: BatchClientRun) -> Dict[str, Any]: for detail in run.result.details: if detail.error and detail.error.exception: per_line_errors[detail.index] = str(detail.error.exception) - if isinstance(detail.error.exception, DecisionEvaluationException): - per_line_error_metadata[detail.index] = _decision_error_metadata(detail.error.exception) + exception = detail.error.exception + if ( + isinstance(exception, EvaluationException) + and isinstance(exception.properties, dict) + and isinstance(exception.properties.get("decision_model"), dict) + ): + per_line_error_metadata[detail.index] = _decision_error_metadata(exception) summary = { "status": run.status.value, diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/__init__.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/__init__.py index fd90b6dd5bbf..aa848b355e25 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/__init__.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/__init__.py @@ -4,7 +4,6 @@ from ._base_eval import EvaluatorBase from ._base_model_eval import ModelEvaluatorBase -from ._decision_execution import DecisionExecutionPolicy, DecisionRequestObservation from ._base_prompty_eval import PromptyEvaluatorBase from ._base_rai_svc_eval import RaiServiceEvaluatorBase from ._base_multi_eval import MultiEvaluatorBase @@ -13,8 +12,6 @@ __all__ = [ "EvaluatorBase", "ModelEvaluatorBase", - "DecisionExecutionPolicy", - "DecisionRequestObservation", "PromptyEvaluatorBase", "RaiServiceEvaluatorBase", "MultiEvaluatorBase", diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py index 7b500cae49e8..63345e0e4416 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py @@ -3,13 +3,12 @@ # --------------------------------------------------------- import math -from typing import Any, Dict, List, Optional, TypeVar, Union +from typing import Any, Callable, Dict, List, Optional, TypeVar, Union from azure.core.credentials import TokenCredential from azure.ai.evaluation._constants import EVALUATION_PASS_FAIL_MAPPING from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget from . import EvaluatorBase -from ._decision_execution import DecisionExecutionPolicy from ._model_runners import ( DecisionModelRunner, PromptyModelRunner, @@ -49,9 +48,21 @@ class ModelEvaluatorBase(EvaluatorBase[T]): :keyword is_reasoning_model: This parameter is in preview. If True, updates the config parameters in prompty file based on reasoning models. Defaults to False. :type is_reasoning_model: bool - :keyword decision_execution_policy: Optional transport-only limits and receipt callbacks for a decision runner. - Rejected for Prompty evaluators; omitted policies preserve existing Prompty behavior. - :paramtype decision_execution_policy: Optional[DecisionExecutionPolicy] + :keyword timeout: Optional overall Decision time limit, including authentication and retries. + :paramtype timeout: Optional[float] + :keyword timeout_seconds: Optional Decision connect/read timeout. + :paramtype timeout_seconds: Optional[float] + :keyword max_retries: Optional Decision retry limit, defaults to 3. + :paramtype max_retries: Optional[int] + :keyword cancellation_requested: Optional Decision cancellation callback, polled during authentication, + requests and retry waits. Cancelling stops awaiting synchronous credential calls, not their worker threads. + :paramtype cancellation_requested: Optional[Callable[[], bool]] + :keyword max_request_bytes: Optional limit on the compact UTF-8 serialized Decision request. + :paramtype max_request_bytes: Optional[int] + :keyword max_state_question_bytes: Optional limit on the largest individual state/question pair. + :paramtype max_state_question_bytes: Optional[int] + :keyword max_response_bytes: Optional response acceptance limit before JSON parsing, not a streaming bound. + :paramtype max_response_bytes: Optional[int] """ _LLM_CALL_TIMEOUT = 600 @@ -71,10 +82,23 @@ def __init__( eval_last_turn: bool = False, threshold: int = 3, credential: Optional[TokenCredential] = None, - decision_execution_policy: Optional[DecisionExecutionPolicy] = None, + timeout: Optional[float] = None, + timeout_seconds: Optional[float] = None, + max_retries: Optional[int] = None, + cancellation_requested: Optional[Callable[[], bool]] = None, + max_request_bytes: Optional[int] = None, + max_state_question_bytes: Optional[int] = None, + max_response_bytes: Optional[int] = None, _higher_is_better: bool = False, **kwargs, ) -> None: + if "decision_execution_policy" in kwargs: + raise EvaluationException( + message="Use explicit Decision execution controls instead of decision_execution_policy.", + blame=ErrorBlame.USER_ERROR, + category=ErrorCategory.INVALID_VALUE, + target=ErrorTarget.EVALUATE, + ) self._result_key = result_key self._is_reasoning_model = kwargs.get("is_reasoning_model", False) self._prompty_file = prompty_file @@ -83,6 +107,14 @@ def __init__( super().__init__(eval_last_turn=eval_last_turn, threshold=threshold, _higher_is_better=_higher_is_better) subclass_name = self.__class__.__name__ + execution_options: Dict[str, Any] = { + "timeout_seconds": timeout_seconds, + "max_retries": max_retries, + "cancellation_requested": cancellation_requested, + "max_request_bytes": max_request_bytes, + "max_state_question_bytes": max_state_question_bytes, + "max_response_bytes": max_response_bytes, + } if is_decision_model_config(model_config): if not self._supports_decision_models: raise EvaluationException( @@ -92,17 +124,31 @@ def __init__( category=ErrorCategory.INVALID_VALUE, target=ErrorTarget.EVALUATE, ) + if timeout is not None: + if ( + isinstance(timeout, bool) + or not isinstance(timeout, (int, float)) + or not math.isfinite(timeout) + or timeout <= 0 + ): + raise EvaluationException( + message="'timeout' must be a positive finite number.", + blame=ErrorBlame.USER_ERROR, + category=ErrorCategory.INVALID_VALUE, + target=ErrorTarget.EVALUATE, + ) + self._LLM_CALL_TIMEOUT = min(self._LLM_CALL_TIMEOUT, timeout) self._model_runner: Union[DecisionModelRunner, PromptyModelRunner] = DecisionModelRunner( model_config, credential, binary=self._binary_decision, evaluator_name=subclass_name, - execution_policy=decision_execution_policy, + **{name: value for name, value in execution_options.items() if value is not None}, ) else: - if decision_execution_policy is not None: + if timeout is not None or any(value is not None for value in execution_options.values()): raise EvaluationException( - message="A decision_execution_policy is only supported by decision model evaluators.", + message="Decision execution controls are only supported by decision model evaluators.", blame=ErrorBlame.USER_ERROR, category=ErrorCategory.INVALID_VALUE, target=ErrorTarget.EVALUATE, diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_execution.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_execution.py deleted file mode 100644 index 56d536f317d5..000000000000 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_execution.py +++ /dev/null @@ -1,81 +0,0 @@ -# --------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# --------------------------------------------------------- - -"""Transport-only execution settings for hosted decision evaluators.""" - -import math -from dataclasses import dataclass -from typing import Any, Callable, Dict, Literal, Optional, Tuple - -from ..._exceptions import EvaluationException - - -@dataclass(frozen=True) -class DecisionRequestObservation: - """A sanitized per-attempt receipt. Unknown usage and model values remain ``None``.""" - - attempt: int - outcome: Literal["response", "error", "timeout", "cancelled", "request_rejected"] - http_status: Optional[int] = None - input_tokens: Optional[int] = None - output_tokens: Optional[int] = None - model: Optional[str] = None - - -@dataclass(frozen=True) -class DecisionExecutionPolicy: - """Internal hosted execution policy, separate from the model configuration. - - ``timeout_seconds`` bounds each transport connect/read; ``deadline_seconds`` caps the entire - operation, including authentication and azure-core retries. Request budgets measure compact UTF-8 JSON, - not estimated tokens. ``max_state_question_bytes`` bounds the largest ``{"state", "question"}`` pair. - ``max_response_bytes`` limits the body accepted for JSON parsing, not transport buffering or allocation. - Cancellation is polled every 50 ms, including during authentication and retry backoff. - Cancellation and deadlines stop awaiting authentication, but cannot forcibly interrupt a synchronous - credential call already running in a worker thread or bound that thread's shutdown time. - The synchronous observer receives transport receipts only; callback failures are explicit errors. - """ - - timeout_seconds: Optional[float] = None - deadline_seconds: Optional[float] = None - max_retries: int = 3 - max_request_bytes: Optional[int] = None - max_state_question_bytes: Optional[int] = None - max_response_bytes: Optional[int] = None - cancellation_requested: Optional[Callable[[], bool]] = None - observer: Optional[Callable[[DecisionRequestObservation], None]] = None - - def __post_init__(self) -> None: - for name in ("timeout_seconds", "deadline_seconds"): - value = getattr(self, name) - if value is not None and ( - isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value <= 0 - ): - raise ValueError(f"{name} must be a positive finite number.") - for name in ("max_request_bytes", "max_state_question_bytes", "max_response_bytes"): - value = getattr(self, name) - if value is not None and (isinstance(value, bool) or not isinstance(value, int) or value <= 0): - raise ValueError(f"{name} must be a positive integer.") - if isinstance(self.max_retries, bool) or not isinstance(self.max_retries, int) or self.max_retries < 0: - raise ValueError("max_retries must be a nonnegative integer.") - for name in ("cancellation_requested", "observer"): - value = getattr(self, name) - if value is not None and not callable(value): - raise ValueError(f"{name} must be callable.") - - -class DecisionEvaluationException(EvaluationException): - """An execution or interpretation error retaining only known, sanitized decision metadata.""" - - def __init__( - self, - message: str, - *, - observations: Tuple[DecisionRequestObservation, ...] = (), - properties: Optional[Dict[str, Any]] = None, - **kwargs: Any, - ) -> None: - self.observations = tuple(observations) - self.properties = properties - super().__init__(message, **kwargs) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py index 04517c6169cc..4f44d2ab6a8d 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py @@ -13,9 +13,8 @@ from dataclasses import dataclass from typing import Any, Callable, Dict, List, Optional, Tuple, TypeVar, Union -from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget +from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException from ._base_model_eval import ModelEvaluatorBase, not_applicable_result -from ._decision_execution import DecisionEvaluationException from ._model_runners import DecisionModelRunner, validate_noul_answer, validate_score_answer STATUS_CUTOFF = 0.5 @@ -27,7 +26,7 @@ def _checked_value(validate: Callable[..., T], error_target: ErrorTarget, *args: return validate(*args) except ValueError as exc: message = f"The decision model returned an invalid response: {exc}" - raise DecisionEvaluationException( + raise EvaluationException( message=message, internal_message=message, blame=ErrorBlame.SYSTEM_ERROR, @@ -175,6 +174,10 @@ async def evaluate(self, eval_input: Dict[str, Any]) -> Dict[str, Any]: total_tokens=tokens["total_token_count"], model=tokens["model_id"], ) + usage_fields = { + f"{key}_{name}": properties[name] + for name in ("prompt_tokens", "completion_tokens", "total_tokens", "model") + } try: skip_reason = decision_skip_reason(response, STATUS_CUTOFF, self.error_target) properties["decision_model"]["status_completion_probability"] = decision_probability( @@ -184,9 +187,10 @@ async def evaluate(self, eval_input: Dict[str, Any]) -> Dict[str, Any]: if skip_reason: result = not_applicable_result(key, skip_reason, self.threshold) result[f"{key}_properties"] = properties + result.update(usage_fields) return result score, reason, details = self.scoring.interpret(response, key, self.error_target) - except DecisionEvaluationException as exc: + except EvaluationException as exc: exc.properties = properties raise properties["decision_model"].update(details) @@ -202,4 +206,5 @@ async def evaluate(self, eval_input: Dict[str, Any]) -> Dict[str, Any]: f"{key}_status": "completed", f"{key}_threshold": self.threshold, f"{key}_properties": properties, + **usage_fields, } diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py index 75c19f9031fc..086477a2562e 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py @@ -13,14 +13,13 @@ import json import math import os -from typing import Any, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional -from azure.core.pipeline.policies import AsyncRedirectPolicy, AsyncRetryPolicy, CustomHookPolicy +from azure.core.pipeline.policies import AsyncRedirectPolicy, AsyncRetryPolicy from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException from ..._common.utils import construct_prompty_model_config, validate_model_config from ..._http_utils import get_async_http_client -from ._decision_execution import DecisionEvaluationException, DecisionExecutionPolicy, DecisionRequestObservation if os.getenv("AI_EVALS_USE_PF_PROMPTY", "false").lower() == "true": from promptflow.core._flow import AsyncPrompty @@ -103,8 +102,8 @@ def _invalid_config(message: str) -> EvaluationException: ) -def _failed(message: str, blame: ErrorBlame = ErrorBlame.SYSTEM_ERROR) -> DecisionEvaluationException: - return DecisionEvaluationException( +def _failed(message: str, blame: ErrorBlame = ErrorBlame.SYSTEM_ERROR) -> EvaluationException: + return EvaluationException( message=message, internal_message=message, blame=blame, @@ -155,31 +154,6 @@ def validate_score_answer(answer: Dict[str, Any], levels: int) -> List[float]: return distribution -class _DecisionRetryPolicy(AsyncRetryPolicy): - def increment(self, settings: Dict[str, Any], response: Any = None, error: Optional[Exception] = None) -> bool: - # EvaluationException is an AzureError, but failed callbacks must not replay an already billed request. - if isinstance(error, EvaluationException): - return False - return super().increment(settings, response=response, error=error) - - -class _DecisionObservationPolicy(CustomHookPolicy): - def __init__(self, runner: "DecisionModelRunner", observations: List[DecisionRequestObservation]) -> None: - super().__init__() - self.runner = runner - self.observations = observations - self.attempt = 0 - - def on_request(self, request: Any) -> None: - self.attempt += 1 - - def on_response(self, request: Any, response: Any) -> None: - self.runner._observe(self.runner._response_observation(response.http_response, self.attempt), self.observations) - - def on_exception(self, request: Any) -> None: - self.runner._observe(DecisionRequestObservation(self.attempt, "error"), self.observations) - - class DecisionModelRunner: """Sends prepared evidence and typed questions to a Microsoft-Decision-1 deployment in Foundry. @@ -188,6 +162,13 @@ class DecisionModelRunner: present, otherwise an Entra credential (from ``credential`` in the configuration or the constructor, or the SDK default). ``api_version`` is accepted and ignored. ``probability_threshold`` (default 0.75) is only accepted when ``binary=True``. + + Execution controls are keyword-only and separate from model configuration. ``timeout_seconds`` bounds + each connect/read; the call's ``timeout`` bounds authentication, retries and the entire operation. + ``max_retries`` defaults to 3. ``cancellation_requested`` is polled every 50 ms, including retry waits; + it stops awaiting synchronous credentials but cannot stop their worker threads. Byte limits measure + compact unescaped UTF-8 JSON, with ``max_state_question_bytes`` applying to the largest individual pair. + ``max_response_bytes`` limits acceptance before parsing, not transport allocation. """ def __init__( @@ -197,11 +178,37 @@ def __init__( *, binary: bool = False, evaluator_name: str = "DecisionModelRunner", - execution_policy: Optional[DecisionExecutionPolicy] = None, + timeout_seconds: Optional[float] = None, + max_retries: int = 3, + cancellation_requested: Optional[Callable[[], bool]] = None, + max_request_bytes: Optional[int] = None, + max_state_question_bytes: Optional[int] = None, + max_response_bytes: Optional[int] = None, ) -> None: - if execution_policy is not None and not isinstance(execution_policy, DecisionExecutionPolicy): - raise _invalid_config("'execution_policy' must be a DecisionExecutionPolicy.") - self._execution_policy = execution_policy or DecisionExecutionPolicy() + if timeout_seconds is not None and ( + isinstance(timeout_seconds, bool) + or not isinstance(timeout_seconds, (int, float)) + or not math.isfinite(timeout_seconds) + or timeout_seconds <= 0 + ): + raise _invalid_config("'timeout_seconds' must be a positive finite number.") + if isinstance(max_retries, bool) or not isinstance(max_retries, int) or max_retries < 0: + raise _invalid_config("'max_retries' must be a nonnegative integer.") + for name, value in ( + ("max_request_bytes", max_request_bytes), + ("max_state_question_bytes", max_state_question_bytes), + ("max_response_bytes", max_response_bytes), + ): + if value is not None and (isinstance(value, bool) or not isinstance(value, int) or value <= 0): + raise _invalid_config(f"'{name}' must be a positive integer.") + if cancellation_requested is not None and not callable(cancellation_requested): + raise _invalid_config("'cancellation_requested' must be callable.") + self._timeout_seconds = timeout_seconds + self._max_retries = max_retries + self._cancellation_requested = cancellation_requested + self._max_request_bytes = max_request_bytes + self._max_state_question_bytes = max_state_question_bytes + self._max_response_bytes = max_response_bytes if not is_decision_model_config(model_config): raise _invalid_config(f"A decision model configuration must be a dict with type '{DECISION_MODEL_TYPE}'.") unexpected = set(model_config) - _DECISION_CONFIG_KEYS - _IGNORED_CONFIG_KEYS @@ -252,12 +259,17 @@ async def __call__(self, *, state: Any, questions: Dict[str, Dict[str, Any]], ti :return: ``{"model", "answers", "usage"}`` with the response's structure checked against the questions. :rtype: Dict[str, Any] """ + if ( + isinstance(timeout, bool) + or not isinstance(timeout, (int, float)) + or not math.isfinite(timeout) + or timeout <= 0 + ): + raise _invalid_config("'timeout' must be a positive finite number.") for question in questions.values(): if question.get("type") not in ("noul", "score"): raise _invalid_config(f"Decision question type '{question.get('type')}' is not supported.") - policy = self._execution_policy - observations: List[DecisionRequestObservation] = [] - observation_policy = _DecisionObservationPolicy(self, observations) + properties = self._response_properties(None) try: body = {"model": self._deployment, "state": state, "questions": questions} try: @@ -277,43 +289,29 @@ async def __call__(self, *, state: Any, questions: Dict[str, Dict[str, Any]], ti default=0, ) except (ValueError, TypeError, OverflowError, UnicodeError): - self._observe(DecisionRequestObservation(0, "request_rejected"), observations) raise _failed( "Decision evidence must be finite JSON-serializable data.", ErrorBlame.USER_ERROR ) from None - if (policy.max_request_bytes is not None and len(content) > policy.max_request_bytes) or ( - policy.max_state_question_bytes is not None and state_question_bytes > policy.max_state_question_bytes + if (self._max_request_bytes is not None and len(content) > self._max_request_bytes) or ( + self._max_state_question_bytes is not None and state_question_bytes > self._max_state_question_bytes ): - self._observe(DecisionRequestObservation(0, "request_rejected"), observations) raise _failed("The decision request exceeds the configured byte budget.", ErrorBlame.USER_ERROR) - deadline = min(timeout, policy.deadline_seconds) if policy.deadline_seconds is not None else timeout try: - payload = await asyncio.wait_for(self._await_request(content, observation_policy), timeout=deadline) + payload = await asyncio.wait_for(self._await_request(content), timeout=timeout) except asyncio.TimeoutError: - self._observe(DecisionRequestObservation(observation_policy.attempt, "timeout"), observations) raise _failed("The decision model request timed out.") from None + properties = self._response_properties(payload) try: return self._validate_response(payload, questions) except ValueError as exc: raise _failed(f"The decision model returned an invalid response: {exc}") from None - except DecisionEvaluationException as exc: - if not observations: - self._observe(DecisionRequestObservation(0, "error"), observations) - exc.observations = tuple(observations) + except EvaluationException as exc: + if exc.properties is None: + exc.properties = properties raise - def _observe(self, observation: DecisionRequestObservation, observations: List[DecisionRequestObservation]) -> None: - observations.append(observation) - if self._execution_policy.observer is not None: - try: - self._execution_policy.observer(observation) - except Exception: # pylint: disable=broad-except - error = _failed("The decision transport observer failed.", ErrorBlame.USER_ERROR) - error.observations = tuple(observations) - raise error from None - def _cancelled(self) -> bool: - callback = self._execution_policy.cancellation_requested + callback = self._cancellation_requested if callback is None: return False try: @@ -324,18 +322,15 @@ def _cancelled(self) -> bool: raise _failed("The decision cancellation callback must return a boolean.", ErrorBlame.USER_ERROR) return result - async def _await_request(self, content: bytes, observation_policy: _DecisionObservationPolicy) -> Any: - observations = observation_policy.observations + async def _await_request(self, content: bytes) -> Any: if self._cancelled(): - self._observe(DecisionRequestObservation(0, "cancelled"), observations) raise _failed("The decision model request was cancelled.", ErrorBlame.USER_ERROR) - if self._execution_policy.cancellation_requested is None: - return await self._post(content, observation_policy) - task = asyncio.create_task(self._post(content, observation_policy)) + if self._cancellation_requested is None: + return await self._post(content) + task = asyncio.create_task(self._post(content)) try: while not task.done(): if self._cancelled(): - self._observe(DecisionRequestObservation(observation_policy.attempt, "cancelled"), observations) raise _failed("The decision model request was cancelled.", ErrorBlame.USER_ERROR) await asyncio.wait({task}, timeout=0.05) return await task @@ -361,8 +356,7 @@ async def _get_default_token() -> str: async with AsyncAzureTokenProvider() as provider: return (await provider.get_token(_DECISION_TOKEN_SCOPE)).token - async def _post(self, content: bytes, observation_policy: _DecisionObservationPolicy) -> Any: - observations = observation_policy.observations + async def _post(self, content: bytes) -> Any: headers = {"x-ms-useragent": self._user_agent, "Content-Type": "application/json"} if self._api_key: headers["api-key"] = self._api_key @@ -375,77 +369,63 @@ async def _post(self, content: bytes, observation_policy: _DecisionObservationPo raise _failed("Could not acquire a token for the decision model.", ErrorBlame.USER_ERROR) from None policies: Dict[str, Any] = { - "retry_policy": _DecisionRetryPolicy( - retry_total=self._execution_policy.max_retries, - retry_connect=self._execution_policy.max_retries, - retry_read=self._execution_policy.max_retries, - retry_status=self._execution_policy.max_retries, + "retry_policy": AsyncRetryPolicy( + retry_total=self._max_retries, + retry_connect=self._max_retries, + retry_read=self._max_retries, + retry_status=self._max_retries, retry_backoff_factor=0.8, retry_on_status_codes=_DECISION_RETRY_STATUS_CODES, ), "redirect_policy": AsyncRedirectPolicy(permit_redirects=False), } options: Dict[str, Any] = {"retry_on_methods": ["POST"]} - if self._execution_policy.timeout_seconds is not None: - options.update( - connection_timeout=self._execution_policy.timeout_seconds, - read_timeout=self._execution_policy.timeout_seconds, - ) + if self._timeout_seconds is not None: + # The default requests transport uses this scalar for both connect and read timeouts. + options["connection_timeout"] = self._timeout_seconds - policies["custom_hook_policy"] = observation_policy try: async with get_async_http_client().with_policies(**policies) as client: response = await client.post(self._url, content=content, headers=headers, **options) status = response.status_code - if not observations: - self._observe(self._response_observation(response, 1), observations) - content = response.body() if status == 200 else b"" + content = response.body() except EvaluationException: raise except Exception: # pylint: disable=broad-except - if not observations or observations[-1].outcome != "error": - self._observe(DecisionRequestObservation(observation_policy.attempt or 1, "error"), observations) raise _failed("The decision model request failed; check the endpoint and network access.") from None - if status != 200: - blame = ErrorBlame.USER_ERROR if status in (400, 401, 403, 404, 422) else ErrorBlame.SYSTEM_ERROR - raise _failed(f"The decision model returned HTTP {status}.", blame) - if ( - self._execution_policy.max_response_bytes is not None - and len(content) > self._execution_policy.max_response_bytes - ): - raise _failed("The decision response exceeds the configured byte budget.") - try: - return json.loads(content) - except (ValueError, TypeError): - raise _failed("The decision model returned invalid JSON.") from None - - def _response_observation(self, response: Any, attempt: int) -> DecisionRequestObservation: - content = response.body() - payload: Any = None - if ( - self._execution_policy.max_response_bytes is None - or len(content) <= self._execution_policy.max_response_bytes - ): + if self._max_response_bytes is not None and len(content) > self._max_response_bytes: + if status == 200: + raise _failed("The decision response exceeds the configured byte budget.") + payload = None + else: try: payload = json.loads(content) except (ValueError, TypeError): - pass + if status == 200: + raise _failed("The decision model returned invalid JSON.") from None + payload = None + if status != 200: + blame = ErrorBlame.USER_ERROR if status in (400, 401, 403, 404, 422) else ErrorBlame.SYSTEM_ERROR + error = _failed(f"The decision model returned HTTP {status}.", blame) + error.properties = self._response_properties(payload) + raise error + return payload + + def _response_properties(self, payload: Any) -> Dict[str, Any]: + """Retain only independently known final-response usage, never raw answers or retry totals.""" + properties: Dict[str, Any] = {"decision_model": {"deployment": self._deployment}} usage = payload.get("usage") if isinstance(payload, dict) else None usage = usage if isinstance(usage, dict) else {} - - def token_count(name: str) -> Optional[int]: - value = usage.get(name) - return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None - + for native_key, key in (("input_tokens", "prompt_tokens"), ("output_tokens", "completion_tokens")): + value = usage.get(native_key) + if isinstance(value, int) and not isinstance(value, bool) and value >= 0: + properties[key] = value + if "prompt_tokens" in properties and "completion_tokens" in properties: + properties["total_tokens"] = properties["prompt_tokens"] + properties["completion_tokens"] model = payload.get("model") if isinstance(payload, dict) else None - return DecisionRequestObservation( - attempt=attempt, - outcome="response", - http_status=response.status_code, - input_tokens=token_count("input_tokens"), - output_tokens=token_count("output_tokens"), - model=model if isinstance(model, str) and model.strip() else None, - ) + if isinstance(model, str) and model.strip(): + properties["model"] = model + return properties @staticmethod def _validate_response(payload: Any, questions: Dict[str, Dict[str, Any]]) -> Dict[str, Any]: diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_exceptions.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_exceptions.py index c73dc13bb15a..c499f7196cf7 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_exceptions.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_exceptions.py @@ -4,7 +4,7 @@ """This includes enums and classes for exceptions for use in azure-ai-evaluation.""" from enum import Enum -from typing import Optional +from typing import Any, Dict, Optional from azure.core.exceptions import AzureError @@ -129,6 +129,8 @@ class EvaluationException(AzureError): :type balance: ~azure.ai.evaluation._exceptions.ErrorBlame :param tsg_link: A link to the TSG page for troubleshooting the error. :type tsg_link: str + :param properties: Optional known evaluator metadata retained for failed-row persistence. + :type properties: Optional[Dict[str, Any]] """ def __init__( @@ -140,6 +142,7 @@ def __init__( category: ErrorCategory = ErrorCategory.UNKNOWN, blame: ErrorBlame = ErrorBlame.UNKNOWN, tsg_link: Optional[str] = None, + properties: Optional[Dict[str, Any]] = None, **kwargs, ) -> None: self.category = category @@ -147,6 +150,7 @@ def __init__( self.blame = blame self.internal_message = internal_message self.tsg_link = tsg_link + self.properties = properties super().__init__(message, *args, **kwargs) def __str__(self): diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_http_utils.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_http_utils.py index 163f73a8f426..0b0448f5a903 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_http_utils.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_http_utils.py @@ -249,20 +249,6 @@ def __enter__(self) -> Self: return cast(Self, super().__enter__()) -class _AsyncioRequestsTransportWithReadTimeout(AsyncioRequestsTransport): - """Adapt explicit read timeouts for the requests-backed default transport.""" - - async def send(self, request, **kwargs): - # Translate after RetryPolicy has bounded the numeric connection timeout. - # AsyncioRequestsTransport otherwise forwards read_timeout to requests, which rejects it. - if "read_timeout" in kwargs: - connection_timeout = kwargs.pop("connection_timeout", self.connection_config.timeout) - if isinstance(connection_timeout, tuple): - raise ValueError("Cannot set tuple connection_timeout and read_timeout together") - kwargs["connection_timeout"] = (connection_timeout, kwargs.pop("read_timeout")) - return await super().send(request, **kwargs) - - class AsyncHttpPipeline(AsyncPipeline): """A *very* thin wrapper over azure.core.pipeline.AsyncPipeline that facilitates sending miscellaneous http requests by adding: @@ -337,7 +323,7 @@ def __init__( # AsyncioRequestsTransport normally should not be imported outside of azure.core, since transports # are meant to be user configurable. # AsyncioRequestsTransport is only used in this file as the default transport when not user specified. - transport=transport or _AsyncioRequestsTransportWithReadTimeout(**kwargs), + transport=transport or AsyncioRequestsTransport(**kwargs), policies=[ config.headers_policy, config.user_agent_policy, diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py index dd1f8ecad78d..e2e958e9111e 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py @@ -14,9 +14,10 @@ from azure.ai.evaluation import F1ScoreEvaluator from azure.ai.evaluation._evaluate._batch_run import RunSubmitterClient -from azure.ai.evaluation._evaluate._evaluate import _convert_single_row_to_aoai_format -from azure.ai.evaluation._evaluators._common import DecisionExecutionPolicy, DecisionRequestObservation -from azure.ai.evaluation._evaluators._common._decision_execution import DecisionEvaluationException +from azure.ai.evaluation._evaluate._evaluate import ( + _calculate_aoai_evaluation_summary, + _convert_single_row_to_aoai_format, +) from azure.ai.evaluation._evaluators._common._decision_strategy import BinaryScoring, DecisionEvaluationStrategy from azure.ai.evaluation._evaluators._common._model_runners import DecisionModelRunner from azure.ai.evaluation._evaluators._common import _model_runners @@ -126,6 +127,17 @@ async def judge(query): assert len(result["properties"]["decision_model"]["rubric_hash"]) == 64 assert "native_status_answer" not in result["properties"]["decision_model"] assert summary["judge"]["failed_lines"] == 1 + rollup = _calculate_aoai_evaluation_summary(rows, logging.getLogger("test"), CRITERIA) + assert rollup["per_model_usage"] == [ + { + "model_name": "decision-native", + "invocation_count": 1, + "prompt_tokens": 120, + "completion_tokens": 3, + "total_tokens": 123, + "cached_tokens": 0, + } + ] @pytest.mark.parametrize("status, expected", [(0.9, "completed"), (0.1, "skipped")]) def test_success_and_skip_are_not_stamped_as_errors(self, batch_client, tmp_path, status, expected): @@ -143,11 +155,29 @@ async def judge(query): assert result["status"] == expected assert "error" not in (result.get("sample") or {}) assert "per_line_error_metadata" not in summary["judge"] + assert result["sample"]["usage"] == {"prompt_tokens": 120, "completion_tokens": 3, "total_tokens": 123} + assert result["sample"]["model"] == "decision-native" + assert result["properties"]["total_tokens"] == 123 + if expected == "skipped": + assert result["score"] is None and result["passed"] is None + else: + assert result["score"] == 1 and result["passed"] is True + rollup = _calculate_aoai_evaluation_summary(rows, logging.getLogger("test"), CRITERIA) + assert rollup["per_model_usage"] == [ + { + "model_name": "decision-native", + "invocation_count": 1, + "prompt_tokens": 120, + "completion_tokens": 3, + "total_tokens": 123, + "cached_tokens": 0, + } + ] @pytest.mark.parametrize( "case", ["malformed_shape", "invalid_json", "partial_usage", "precancel", "inflight_cancel", "billed_cancel"] ) - def test_real_pipeline_failure_receipts_survive_batch_conversion(self, batch_client, tmp_path, monkeypatch, case): + def test_real_pipeline_failure_metadata_survives_batch_conversion(self, batch_client, tmp_path, monkeypatch, case): cancelled = False class Response(HttpResponse): @@ -178,17 +208,15 @@ async def __aexit__(self, *_): await self.close() async def send(self, request, **kwargs): + nonlocal cancelled if case == "inflight_cancel": await asyncio.sleep(10) + if case == "billed_cancel": + cancelled = True return Response(request) monkeypatch.setattr(_model_runners, "get_async_http_client", lambda: AsyncHttpPipeline(transport=Transport())) - def observer(receipt): - nonlocal cancelled - if case == "billed_cancel": - cancelled = True - async def judge(query): nonlocal cancelled cancelled = case == "precancel" @@ -203,11 +231,8 @@ def cancel(): try: runner = DecisionModelRunner( CONFIG, - execution_policy=DecisionExecutionPolicy( - max_retries=1, - cancellation_requested=lambda: cancelled, - observer=observer, - ), + max_retries=1, + cancellation_requested=lambda: cancelled, ) return await strategy(runner).evaluate({"query": query}) finally: @@ -217,21 +242,21 @@ def cancel(): _, rows = convert_batch(batch_client, judge, tmp_path) result = next(item for item in rows[0]["results"] if item["name"] == "judge") assert result["status"] == "error" and result["score"] is None and result["passed"] is None - receipts = result["properties"]["decision_request_observations"] - assert all( - set(receipt) == {"attempt", "outcome", "http_status", "input_tokens", "output_tokens", "model"} - for receipt in receipts - ) - assert "usage" not in result["sample"] and "model" not in result["sample"] - if case in ("malformed_shape", "partial_usage", "billed_cancel"): - assert receipts[0]["input_tokens"] == (None if case == "partial_usage" else 12) - assert receipts[0]["output_tokens"] == 4 and receipts[0]["model"] == "decision-native" + assert "decision_request_observations" not in result["properties"] + if case == "malformed_shape": + assert result["sample"]["usage"] == {"prompt_tokens": 12, "completion_tokens": 4, "total_tokens": 16} + assert result["sample"]["model"] == "decision-native" + elif case == "partial_usage": + assert result["sample"]["usage"] == {"completion_tokens": 4} + assert result["sample"]["model"] == "decision-native" + assert "total_tokens" not in result["properties"] and "prompt_tokens" not in result["properties"] else: - assert receipts[0]["input_tokens"] is None and receipts[0]["output_tokens"] is None + assert "usage" not in result["sample"] and "model" not in result["sample"] + assert result["properties"] == {"decision_model": {"deployment": "Decision-1"}} if "cancel" in case: assert result["sample"]["error"]["blame"] == "UserError" - assert receipts[-1]["outcome"] == "cancelled" - assert receipts[-1]["attempt"] == (0 if case == "precancel" else 1) + else: + assert result["sample"]["error"]["blame"] == "SystemError" assert "test-key-not-for-persistence" not in json.dumps(result) assert "synthetic" not in json.dumps(result["properties"]) @@ -243,14 +268,14 @@ async def judge(query): index = calls calls += 1 if index == 0: - raise DecisionEvaluationException( + raise EvaluationException( "Cancelled.", category=ErrorCategory.FAILED_EXECUTION, blame=ErrorBlame.USER_ERROR, - observations=(DecisionRequestObservation(0, "cancelled"),), + properties={"decision_model": {}}, ) if index == 1: - raise DecisionEvaluationException( + raise EvaluationException( "Invalid.", category=ErrorCategory.INVALID_VALUE, blame=ErrorBlame.SYSTEM_ERROR, diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_default_transport.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_default_transport.py index f26088f92f90..24f723e8a807 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_default_transport.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_default_transport.py @@ -14,9 +14,8 @@ import requests from urllib3.util import connection -from azure.ai.evaluation._evaluators._common import DecisionExecutionPolicy from azure.ai.evaluation._evaluators._common._model_runners import DecisionModelRunner -from azure.ai.evaluation._exceptions import EvaluationException +from azure.ai.evaluation._exceptions import ErrorBlame, EvaluationException from azure.ai.evaluation._http_utils import get_async_http_client from azure.core.pipeline.policies import AsyncRetryPolicy @@ -64,10 +63,10 @@ def do_POST(self): thread.join() -def invoke(endpoint, policy, timeout=3): +def invoke(endpoint, timeout=3, **controls): runner = DecisionModelRunner( {"type": "decision", "azure_endpoint": endpoint, "azure_deployment": "Decision-1", "api_key": "local-test"}, - execution_policy=policy, + **controls, ) return asyncio.run(runner(state={"query": "é"}, questions=QUESTIONS, timeout=timeout)) @@ -91,13 +90,10 @@ def record(session, *args, **kwargs): monkeypatch.setattr(requests.Session, "request", record) monkeypatch.setattr(connection, "create_connection", record_connect) - receipts = [] - result = invoke(endpoint, DecisionExecutionPolicy(timeout_seconds=0.5, max_retries=1, observer=receipts.append)) + result = invoke(endpoint, timeout_seconds=0.5, max_retries=1) assert result["usage"] == {"input_tokens": 12, "output_tokens": 4} - assert observed_timeouts == [(0.5, 0.5), (0.5, 0.5)] + assert observed_timeouts == [0.5, 0.5] assert socket_connect_timeouts == [0.5, 0.5] - assert [receipt.http_status for receipt in receipts] == [429, 200] - assert [receipt.attempt for receipt in receipts] == [1, 2] assert len(calls) == 2 and calls[0]["body"] == calls[1]["body"] assert calls[0]["path"] == "/providers/microsoft/v1/systemone" assert calls[0]["content_type"] == "application/json" @@ -110,14 +106,13 @@ def test_default_transport_read_timeout_is_enforced(localhost): delay[0] = 0.5 started = time.monotonic() with pytest.raises(EvaluationException) as raised: - invoke(endpoint, DecisionExecutionPolicy(timeout_seconds=0.05, max_retries=0)) + invoke(endpoint, timeout_seconds=0.05, max_retries=0) assert time.monotonic() - started < 0.4 assert len(calls) == 1 - assert raised.value.observations[0].attempt == 1 - assert raised.value.observations[0].outcome == "error" + assert raised.value.properties == {"decision_model": {"deployment": "Decision-1"}} -def test_default_transport_numeric_retry_deadline_and_separate_timeout_bounds(localhost, monkeypatch): +def test_default_transport_native_scalar_timeout_preserves_numeric_retry_deadline(localhost, monkeypatch): endpoint, _, _, _ = localhost original = requests.Session.request observed = [] @@ -130,11 +125,10 @@ def record(session, *args, **kwargs): async def request(): async with get_async_http_client(retry_policy=AsyncRetryPolicy(retry_total=0)) as client: - return await client.post(endpoint, content=b"{}", connection_timeout=2, read_timeout=0.3, timeout=0.8) + return await client.post(endpoint, content=b"{}", connection_timeout=2, timeout=0.8) assert asyncio.run(request()).status_code == 200 - assert 0 < observed[0][0] <= 0.8 - assert observed[0][1] == 0.3 + assert 0 < observed[0] <= 0.8 def test_default_transport_without_read_override_preserves_scalar_timeout(localhost, monkeypatch): @@ -154,3 +148,30 @@ async def request(): assert asyncio.run(request()).status_code == 200 assert observed == [0.7] + + +def test_default_transport_cancellation_interrupts_awaiting_a_real_request(localhost): + endpoint, calls, delay, _ = localhost + delay[0] = 0.5 + + async def request(): + cancelled = asyncio.Event() + runner = DecisionModelRunner( + {"type": "decision", "azure_endpoint": endpoint, "azure_deployment": "Decision-1", "api_key": "local-test"}, + timeout_seconds=1, + max_retries=0, + cancellation_requested=cancelled.is_set, + ) + started = time.monotonic() + handle = asyncio.get_running_loop().call_later(0.05, cancelled.set) + try: + with pytest.raises(EvaluationException, match="cancelled") as raised: + await runner(state={}, questions=QUESTIONS, timeout=3) + assert time.monotonic() - started < 0.4 + assert raised.value.blame == ErrorBlame.USER_ERROR + assert raised.value.properties == {"decision_model": {"deployment": "Decision-1"}} + finally: + handle.cancel() + + asyncio.run(request()) + assert len(calls) == 1 diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py index a614a259b51f..546bc5557182 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py @@ -8,7 +8,6 @@ import hashlib import json import threading -from dataclasses import FrozenInstanceError from concurrent.futures import ThreadPoolExecutor from types import SimpleNamespace from unittest.mock import AsyncMock, Mock, patch @@ -20,7 +19,6 @@ from azure.ai.evaluation import FluencyEvaluator from azure.ai.evaluation._evaluators._common import EvaluatorBase, ModelEvaluatorBase, PromptyEvaluatorBase -from azure.ai.evaluation._evaluators._common import DecisionExecutionPolicy, DecisionRequestObservation from azure.ai.evaluation._evaluators._common import _model_runners from azure.ai.evaluation._evaluators._common._decision_strategy import ( BinaryScoring, @@ -255,14 +253,9 @@ async def send(self, request, **kwargs): transport = Transport() monkeypatch.setattr(_model_runners, "get_async_http_client", lambda: AsyncHttpPipeline(transport=transport)) - observations = [] - result = run( - DecisionModelRunner(DECISION_CONFIG, execution_policy=DecisionExecutionPolicy(observer=observations.append)) - ) + result = run(DecisionModelRunner(DECISION_CONFIG)) assert transport.calls == 2 assert result["model"] == "microsoft-decision-1" - assert [receipt.attempt for receipt in observations] == [1, 2] - assert [receipt.http_status for receipt in observations] == [status, 200] def test_default_credential_token_acquisition_runs_off_event_loop(self, client): from azure.ai.evaluation._legacy._common._async_token_provider import AsyncAzureTokenProvider @@ -658,12 +651,11 @@ def test_successful_properties_preserve_common_metadata_shape(self): ) -class TestDecisionExecutionPolicy: - def test_policy_and_receipts_are_immutable(self): - with pytest.raises(FrozenInstanceError): - DecisionExecutionPolicy().max_retries = 4 - with pytest.raises(FrozenInstanceError): - DecisionRequestObservation(1, "response").attempt = 2 +class TestDecisionExecutionControls: + @pytest.mark.parametrize("name", ["execution_policy", "observer", "deadline_seconds"]) + def test_obsolete_controls_are_not_accepted(self, name): + with pytest.raises(TypeError, match=name): + DecisionModelRunner(DECISION_CONFIG, **{name: object()}) @pytest.mark.parametrize( "options", @@ -671,52 +663,85 @@ def test_policy_and_receipts_are_immutable(self): {"timeout_seconds": 0}, {"timeout_seconds": True}, {"timeout_seconds": float("nan")}, - {"deadline_seconds": -1}, - {"deadline_seconds": float("inf")}, {"max_retries": True}, {"max_retries": -1}, {"max_retries": 1.5}, {"max_request_bytes": 0}, {"max_state_question_bytes": False}, {"max_response_bytes": "100"}, - {"observer": "callback"}, {"cancellation_requested": False}, ], ) - def test_invalid_policy(self, options): - with pytest.raises(ValueError): - DecisionExecutionPolicy(**options) + def test_invalid_controls(self, options): + with pytest.raises(EvaluationException) as raised: + DecisionModelRunner(DECISION_CONFIG, **options) + assert raised.value.blame == ErrorBlame.USER_ERROR - def test_base_forwards_policy_outside_strict_config(self): - policy = DecisionExecutionPolicy(max_retries=0) + def test_base_forwards_controls_outside_strict_config(self): evaluator = ProbeModelEvaluator( - result_key="probe", model_config=DECISION_CONFIG, decision_execution_policy=policy + result_key="probe", model_config=DECISION_CONFIG, max_retries=0, timeout=5, timeout_seconds=2 ) - assert evaluator._decision_model_runner._execution_policy is policy + assert evaluator._decision_model_runner._max_retries == 0 + assert evaluator._decision_model_runner._timeout_seconds == 2 + assert evaluator._LLM_CALL_TIMEOUT == 5 with pytest.raises(EvaluationException, match="Unsupported"): - DecisionModelRunner({**DECISION_CONFIG, "decision_execution_policy": policy}) - with pytest.raises(EvaluationException, match="DecisionExecutionPolicy"): - DecisionModelRunner(DECISION_CONFIG, execution_policy={}) + DecisionModelRunner({**DECISION_CONFIG, "max_retries": 0}) + + def test_obsolete_base_policy_cannot_be_silently_ignored(self): + with pytest.raises(EvaluationException, match="explicit Decision"): + ProbeModelEvaluator(result_key="probe", model_config=DECISION_CONFIG, decision_execution_policy=object()) + + @pytest.mark.parametrize("timeout", [0, -1, True, float("nan"), float("inf"), "30"]) + def test_invalid_overall_timeouts_are_rejected(self, client, timeout): + with pytest.raises(EvaluationException, match="positive finite"): + ProbeModelEvaluator(result_key="probe", model_config=DECISION_CONFIG, timeout=timeout) + with pytest.raises(EvaluationException, match="positive finite"): + run(DecisionModelRunner(DECISION_CONFIG), timeout=timeout) + assert not client.calls + + @pytest.mark.parametrize("status", [400, 429, 500]) + @pytest.mark.parametrize("response_limit", [1, 1024]) + def test_final_http_error_keeps_classification_and_only_accepted_known_usage(self, client, status, response_limit): + client.response = FakeResponse(status, json.dumps(level_payload(4)).encode()) + with pytest.raises(EvaluationException, match=f"HTTP {status}") as raised: + run(DecisionModelRunner(DECISION_CONFIG, max_retries=0, max_response_bytes=response_limit)) + assert raised.value.blame == (ErrorBlame.USER_ERROR if status == 400 else ErrorBlame.SYSTEM_ERROR) + if response_limit == 1: + assert raised.value.properties == {"decision_model": {"deployment": "Decision-1"}} + else: + assert raised.value.properties["total_tokens"] == 123 + assert raised.value.properties["model"] == "microsoft-decision-1" - def test_explicit_policy_on_prompty_is_rejected_without_loading_a_flow(self): + @pytest.mark.parametrize( + "controls", + [ + {"timeout": 10}, + {"timeout_seconds": 2}, + {"max_retries": 0}, + {"cancellation_requested": lambda: False}, + {"max_request_bytes": 10}, + {"max_state_question_bytes": 10}, + {"max_response_bytes": 10}, + ], + ) + def test_explicit_controls_on_prompty_are_rejected_without_loading_a_flow(self, controls): with patch.object(_model_runners.AsyncPrompty, "load") as load: with pytest.raises(EvaluationException, match="only supported"): ProbeModelEvaluator( result_key="probe", prompty_file="probe.prompty", model_config=CHAT_CONFIG, - decision_execution_policy=DecisionExecutionPolicy(), + **controls, ) load.assert_not_called() def test_transport_timeout_and_single_retry_policy(self, client): client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) - policy = DecisionExecutionPolicy(timeout_seconds=2, max_retries=7) - run(DecisionModelRunner(DECISION_CONFIG, execution_policy=policy)) + run(DecisionModelRunner(DECISION_CONFIG, timeout_seconds=2, max_retries=7)) retry = client.policies["retry_policy"] assert retry.total_retries == retry.connect_retries == retry.read_retries == retry.status_retries == 7 assert client.calls[0]["options"]["connection_timeout"] == 2 - assert client.calls[0]["options"]["read_timeout"] == 2 + assert "read_timeout" not in client.calls[0]["options"] @pytest.mark.parametrize("field", ["max_request_bytes", "max_state_question_bytes"]) def test_request_budget_exact_boundary_and_no_send_on_overflow(self, client, field): @@ -737,89 +762,70 @@ def test_request_budget_exact_boundary_and_no_send_on_overflow(self, client, fie for question in QUESTIONS.values() ) client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) - run( - DecisionModelRunner(DECISION_CONFIG, execution_policy=DecisionExecutionPolicy(**{field: size})), state=state - ) + run(DecisionModelRunner(DECISION_CONFIG, **{field: size}), state=state) assert b"\xc3\xa9" in client.calls[0]["content"] and b"\\u00e9" not in client.calls[0]["content"] client.calls.clear() with pytest.raises(EvaluationException, match="byte budget") as raised: run( - DecisionModelRunner(DECISION_CONFIG, execution_policy=DecisionExecutionPolicy(**{field: size - 1})), + DecisionModelRunner(DECISION_CONFIG, **{field: size - 1}), state=state, ) assert not client.calls - assert raised.value.observations == (DecisionRequestObservation(0, "request_rejected"),) + assert raised.value.blame == ErrorBlame.USER_ERROR def test_non_json_request_is_rejected_before_authentication_or_network(self, client): with pytest.raises(EvaluationException, match="JSON-serializable") as raised: run(DecisionModelRunner(DECISION_CONFIG), state={"value": float("nan")}) - assert not client.calls and raised.value.observations[0].outcome == "request_rejected" + assert not client.calls and raised.value.blame == ErrorBlame.USER_ERROR def test_response_budget_exact_boundary(self, client): content = json.dumps(level_payload(4)).encode() client.response = FakeResponse(body=content) - run( - DecisionModelRunner( - DECISION_CONFIG, execution_policy=DecisionExecutionPolicy(max_response_bytes=len(content)) - ) - ) + run(DecisionModelRunner(DECISION_CONFIG, max_response_bytes=len(content))) with pytest.raises(EvaluationException, match="byte budget"): - run( - DecisionModelRunner( - DECISION_CONFIG, execution_policy=DecisionExecutionPolicy(max_response_bytes=len(content) - 1) - ) - ) + run(DecisionModelRunner(DECISION_CONFIG, max_response_bytes=len(content) - 1)) def test_structural_failure_preserves_usage_before_answer_validation(self, client): payload = level_payload(4) payload["answers"] = {} client.response = FakeResponse(body=json.dumps(payload).encode()) - receipts = [] with pytest.raises(EvaluationException, match="invalid response") as raised: - run( - DecisionModelRunner(DECISION_CONFIG, execution_policy=DecisionExecutionPolicy(observer=receipts.append)) - ) - assert receipts == list(raised.value.observations) - assert receipts[0].input_tokens == 120 and receipts[0].output_tokens == 3 - assert receipts[0].model == "microsoft-decision-1" + run(DecisionModelRunner(DECISION_CONFIG)) + assert raised.value.properties["prompt_tokens"] == 120 + assert raised.value.properties["completion_tokens"] == 3 + assert raised.value.properties["total_tokens"] == 123 + assert raised.value.properties["model"] == "microsoft-decision-1" def test_unknown_or_invalid_usage_is_not_fabricated(self, client): client.response = FakeResponse(body=json.dumps({"usage": {"input_tokens": True, "output_tokens": 3}}).encode()) with pytest.raises(EvaluationException) as raised: run(DecisionModelRunner(DECISION_CONFIG)) - receipt = raised.value.observations[0] - assert receipt.input_tokens is None and receipt.output_tokens == 3 and receipt.model is None + assert raised.value.properties == {"decision_model": {"deployment": "Decision-1"}, "completion_tokens": 3} - def test_observer_failure_is_explicit_sanitized_and_retains_receipt(self, client): + def test_default_pipeline_has_no_raising_observer_hook(self, client): client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) - observer = Mock(side_effect=RuntimeError("observer-secret")) - with pytest.raises(EvaluationException, match="observer failed") as raised: - run(DecisionModelRunner(DECISION_CONFIG, execution_policy=DecisionExecutionPolicy(observer=observer))) - assert "observer-secret" not in str(raised.value) - assert raised.value.observations[0].input_tokens == 120 + run(DecisionModelRunner(DECISION_CONFIG)) + assert "custom_hook_policy" not in client.policies + assert type(client.policies["retry_policy"]) is AsyncRetryPolicy - def test_deadline_caps_existing_evaluator_timeout(self, client): + def test_base_timeout_caps_existing_evaluator_timeout(self, client): client.delay = 1 + evaluator = ProbeModelEvaluator(result_key="probe", model_config=DECISION_CONFIG, timeout=0.01) with pytest.raises(EvaluationException, match="timed out") as raised: - run(DecisionModelRunner(DECISION_CONFIG, execution_policy=DecisionExecutionPolicy(deadline_seconds=0.01))) - assert raised.value.observations[-1].outcome == "timeout" + run(evaluator._decision_model_runner, timeout=evaluator._LLM_CALL_TIMEOUT) + assert raised.value.blame == ErrorBlame.SYSTEM_ERROR - def test_evaluator_timeout_still_wins_over_longer_policy_deadline(self, client): + def test_evaluator_timeout_still_wins_over_longer_constructor_timeout(self, client): client.delay = 1 + evaluator = ProbeModelEvaluator(result_key="probe", model_config=DECISION_CONFIG, timeout=900) + assert evaluator._LLM_CALL_TIMEOUT == 600 with pytest.raises(EvaluationException, match="timed out"): - run( - DecisionModelRunner(DECISION_CONFIG, execution_policy=DecisionExecutionPolicy(deadline_seconds=5)), - timeout=0.01, - ) + run(evaluator._decision_model_runner, timeout=0.01) def test_precancellation_does_not_send(self, client): with pytest.raises(EvaluationException, match="cancelled") as raised: - run( - DecisionModelRunner( - DECISION_CONFIG, execution_policy=DecisionExecutionPolicy(cancellation_requested=lambda: True) - ) - ) - assert not client.calls and raised.value.observations[-1].outcome == "cancelled" + run(DecisionModelRunner(DECISION_CONFIG, cancellation_requested=lambda: True)) + assert not client.calls and raised.value.blame == ErrorBlame.USER_ERROR def test_inflight_cancellation_cleans_up_request(self, monkeypatch): cleaned_up = [] @@ -843,12 +849,10 @@ async def cancel(): cancellation = asyncio.create_task(cancel()) try: - runner = DecisionModelRunner( - DECISION_CONFIG, execution_policy=DecisionExecutionPolicy(cancellation_requested=lambda: cancelled) - ) + runner = DecisionModelRunner(DECISION_CONFIG, cancellation_requested=lambda: cancelled) with pytest.raises(EvaluationException, match="cancelled") as raised: await runner(state={}, questions=QUESTIONS, timeout=1) - assert raised.value.observations[-1].outcome == "cancelled" + assert raised.value.blame == ErrorBlame.USER_ERROR finally: await cancellation @@ -858,26 +862,29 @@ async def cancel(): @pytest.mark.parametrize("value", [None, "yes", 1]) def test_cancellation_callback_requires_boolean(self, client, value): with pytest.raises(EvaluationException, match="boolean"): - run( - DecisionModelRunner( - DECISION_CONFIG, execution_policy=DecisionExecutionPolicy(cancellation_requested=lambda: value) - ) - ) + run(DecisionModelRunner(DECISION_CONFIG, cancellation_requested=lambda: value)) + assert not client.calls + + def test_cancellation_callback_failure_is_explicit_and_sanitized(self, client): + callback = Mock(side_effect=RuntimeError("cancellation-secret")) + with pytest.raises(EvaluationException, match="cancellation callback failed") as raised: + run(DecisionModelRunner(DECISION_CONFIG, cancellation_requested=callback)) + assert raised.value.blame == ErrorBlame.USER_ERROR + assert "secret" not in str(raised.value) assert not client.calls @pytest.mark.parametrize("field", ["max_request_bytes", "max_state_question_bytes"]) def test_tiny_positive_budget_rejects_before_send(self, client, field): with pytest.raises(EvaluationException, match="byte budget"): - run(DecisionModelRunner(DECISION_CONFIG, execution_policy=DecisionExecutionPolicy(**{field: 1}))) + run(DecisionModelRunner(DECISION_CONFIG, **{field: 1})) assert not client.calls - def test_callback_failure_during_authentication_retains_error_receipt(self, client): + def test_authentication_failure_is_sanitized_and_has_no_fabricated_usage(self, client): config = {key: value for key, value in DECISION_CONFIG.items() if key != "api_key"} credential = Mock(get_token=Mock(side_effect=RuntimeError("credential-secret"))) - observer = Mock(side_effect=RuntimeError("observer-secret")) - with pytest.raises(EvaluationException, match="observer failed") as raised: - run(DecisionModelRunner(config, credential, execution_policy=DecisionExecutionPolicy(observer=observer))) - assert raised.value.observations == (DecisionRequestObservation(0, "error"),) + with pytest.raises(EvaluationException, match="acquire a token") as raised: + run(DecisionModelRunner(config, credential)) + assert raised.value.properties == {"decision_model": {"deployment": "Decision-1"}} assert "secret" not in str(raised.value) assert not client.calls @@ -898,16 +905,14 @@ async def request(): cancel = asyncio.Event() loop = asyncio.get_running_loop() handle = loop.call_later(0.01, cancel.set) - policy = DecisionExecutionPolicy( - deadline_seconds=0.02 if termination == "timeout" else 1, - cancellation_requested=cancel.is_set if termination == "cancelled" else None, - ) try: with pytest.raises(EvaluationException) as raised: - await DecisionModelRunner(config, credential, execution_policy=policy)( - state={}, questions=QUESTIONS, timeout=1 - ) - assert raised.value.observations == (DecisionRequestObservation(0, termination),) + await DecisionModelRunner( + config, credential, cancellation_requested=cancel.is_set if termination == "cancelled" else None + )(state={}, questions=QUESTIONS, timeout=0.02 if termination == "timeout" else 1) + assert raised.value.blame == ( + ErrorBlame.SYSTEM_ERROR if termination == "timeout" else ErrorBlame.USER_ERROR + ) finally: handle.cancel() @@ -934,16 +939,14 @@ async def request(): while not entered.is_set(): await asyncio.sleep(0) handle = loop.call_later(0.01, cancel.set) - policy = DecisionExecutionPolicy( - deadline_seconds=0.03 if termination == "timeout" else 1, - cancellation_requested=cancel.is_set if termination == "cancelled" else None, - ) with patch.object(DecisionModelRunner, "_get_default_token", new_callable=AsyncMock) as get_token: with pytest.raises(EvaluationException) as raised: - await DecisionModelRunner(config, execution_policy=policy)( - state={}, questions=QUESTIONS, timeout=1 - ) - assert raised.value.observations == (DecisionRequestObservation(0, termination),) + await DecisionModelRunner( + config, cancellation_requested=cancel.is_set if termination == "cancelled" else None + )(state={}, questions=QUESTIONS, timeout=0.03 if termination == "timeout" else 1) + assert raised.value.blame == ( + ErrorBlame.SYSTEM_ERROR if termination == "timeout" else ErrorBlame.USER_ERROR + ) get_token.assert_not_called() finally: if handle is not None: @@ -956,7 +959,7 @@ async def request(): @pytest.mark.parametrize("termination", ["timeout", "cancelled"]) @pytest.mark.parametrize("waiting", ["send", "retry_backoff"]) - def test_real_pipeline_termination_retains_actual_attempt(self, monkeypatch, termination, waiting): + def test_real_pipeline_termination_interrupts_send_and_retry_wait(self, monkeypatch, termination, waiting): from azure.ai.evaluation._http_utils import AsyncHttpPipeline cancelled = False @@ -1005,19 +1008,16 @@ def cancel(): cancelled = True handle = loop.call_later(0.01, cancel) - policy = DecisionExecutionPolicy( - deadline_seconds=0.08 if termination == "timeout" else 1, - cancellation_requested=(lambda: cancelled) if termination == "cancelled" else None, - ) try: with pytest.raises(EvaluationException) as raised: - await DecisionModelRunner(DECISION_CONFIG, execution_policy=policy)( - state={}, questions=QUESTIONS, timeout=1 - ) - receipts = raised.value.observations - assert receipts[-1] == DecisionRequestObservation(1, termination) - if waiting == "retry_backoff": - assert receipts[0].http_status == 429 and receipts[0].input_tokens == 120 + await DecisionModelRunner( + DECISION_CONFIG, + cancellation_requested=(lambda: cancelled) if termination == "cancelled" else None, + )(state={}, questions=QUESTIONS, timeout=0.08 if termination == "timeout" else 1) + assert raised.value.blame == ( + ErrorBlame.SYSTEM_ERROR if termination == "timeout" else ErrorBlame.USER_ERROR + ) + assert raised.value.properties == {"decision_model": {"deployment": "Decision-1"}} finally: handle.cancel() @@ -1025,8 +1025,8 @@ def cancel(): assert transport.calls == 1 assert cleaned_up == ([True] if waiting == "send" else []) - @pytest.mark.parametrize("observer_fails", [False, True]) - def test_real_pipeline_transport_errors_retain_each_attempt(self, monkeypatch, observer_fails): + @pytest.mark.parametrize("max_retries", [0, 1]) + def test_real_pipeline_transport_errors_respect_retry_bound(self, monkeypatch, max_retries): from azure.ai.evaluation._http_utils import AsyncHttpPipeline class Transport(AsyncHttpTransport): @@ -1048,19 +1048,14 @@ async def send(self, request, **kwargs): transport = Transport() monkeypatch.setattr(_model_runners, "get_async_http_client", lambda: AsyncHttpPipeline(transport=transport)) - observer = Mock(side_effect=RuntimeError("observer-secret")) if observer_fails else Mock() - policy = DecisionExecutionPolicy(max_retries=1, observer=observer) with pytest.raises(EvaluationException) as raised: - run(DecisionModelRunner(DECISION_CONFIG, execution_policy=policy)) - expected = 1 if observer_fails else 2 + run(DecisionModelRunner(DECISION_CONFIG, max_retries=max_retries)) + expected = max_retries + 1 assert transport.calls == expected - assert raised.value.observations == tuple( - DecisionRequestObservation(i, "error") for i in range(1, expected + 1) - ) - assert observer.call_count == expected + assert raised.value.properties == {"decision_model": {"deployment": "Decision-1"}} assert "secret" not in str(raised.value) - def test_real_pipeline_observer_failure_preserves_response_receipt(self, monkeypatch): + def test_real_pipeline_success_has_no_hook_failure_or_billed_replay(self, monkeypatch): from azure.ai.evaluation._http_utils import AsyncHttpPipeline class Response(HttpResponse): @@ -1073,6 +1068,8 @@ def body(self): return json.dumps(level_payload(4)).encode() class Transport(AsyncHttpTransport): + calls = 0 + async def open(self): pass @@ -1083,13 +1080,10 @@ async def __aexit__(self, *_): await self.close() async def send(self, request, **kwargs): + self.calls += 1 return Response(request) - monkeypatch.setattr(_model_runners, "get_async_http_client", lambda: AsyncHttpPipeline(transport=Transport())) - observer = Mock(side_effect=RuntimeError("observer-secret")) - with pytest.raises(EvaluationException, match="observer failed") as raised: - run(DecisionModelRunner(DECISION_CONFIG, execution_policy=DecisionExecutionPolicy(observer=observer))) - assert raised.value.observations == ( - DecisionRequestObservation(1, "response", 200, 120, 3, "microsoft-decision-1"), - ) - assert observer.call_count == 1 and "secret" not in str(raised.value) + transport = Transport() + monkeypatch.setattr(_model_runners, "get_async_http_client", lambda: AsyncHttpPipeline(transport=transport)) + assert run(DecisionModelRunner(DECISION_CONFIG))["usage"] == {"input_tokens": 120, "output_tokens": 3} + assert transport.calls == 1 From 6acbc6767cfc3e715a102ccc52244d73fa60ed78 Mon Sep 17 00:00:00 2001 From: salma-elshafey Date: Tue, 13 Oct 2026 17:37:10 +0300 Subject: [PATCH 09/24] Parse shared Decision responses and count composite failure usage once Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e564c5c8-480e-48e7-a9e0-44450f774b5e --- .../ai/evaluation/_evaluate/_evaluate.py | 18 ++- .../_evaluators/_common/_decision_strategy.py | 26 +++- .../test_decision_batch_persistence.py | 132 +++++++++++++++++- .../unittests/test_decision_model_support.py | 12 ++ 4 files changed, 175 insertions(+), 13 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py index fbf859e0b589..67d891bfbfc6 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py @@ -3384,6 +3384,12 @@ def _add_error_summaries( metrics = testing_criteria_metadata.get(criteria_name, {}).get("metrics", []) criteria_type = testing_criteria_metadata.get(criteria_name, {}).get("type", "azure_ai_evaluator") + if decision_error_metadata is not None: + evaluator_name = testing_criteria_metadata.get(criteria_name, {}).get("evaluator_name") + if isinstance(evaluator_name, str) and evaluator_name.startswith("builtin."): + primary_metric = evaluator_name.removeprefix("builtin.") + metrics = sorted(metrics, key=lambda metric: metric != primary_metric) + decision_usage_added = False for metric in metrics: if not _should_add_error_summary(run_output_results, criteria_name, metric): @@ -3404,15 +3410,19 @@ def _add_error_summaries( if decision_error_metadata is not None: properties = decision_error_metadata["properties"] error_result["properties"] = properties + # One failed evaluator invocation may produce several metric errors, but only one billed usage. + error_result["sample"] = {"error": error_info} usage = { key: properties[key] for key in ("prompt_tokens", "completion_tokens", "total_tokens") if key in properties } - if usage: - sample["usage"] = usage - if "model" in properties: - sample["model"] = properties["model"] + if not decision_usage_added: + if usage: + error_result["sample"]["usage"] = usage + if "model" in properties: + error_result["sample"]["model"] = properties["model"] + decision_usage_added = True run_output_results.append(error_result) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py index 4f44d2ab6a8d..2a7ef06094cd 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py @@ -156,8 +156,26 @@ def __init__( self.extra_properties = extra_properties async def evaluate(self, eval_input: Dict[str, Any]) -> Dict[str, Any]: - key = self.result_key response = await self.runner(state=dict(eval_input), questions=self.questions, timeout=self.timeout) + result = self.parse(response, eval_input) + tokens = decision_token_counts(response) + result.update( + { + f"{self.result_key}_prompt_tokens": tokens["input_token_count"], + f"{self.result_key}_completion_tokens": tokens["output_token_count"], + f"{self.result_key}_total_tokens": tokens["total_token_count"], + f"{self.result_key}_model": tokens["model_id"], + } + ) + return result + + def parse(self, response: Dict[str, Any], eval_input: Dict[str, Any]) -> Dict[str, Any]: + """Interpret a validated response without making a request. + + Known shared-response usage remains in properties. Only the request-owning result emits standard + usage fields, so composites can parse multiple members without counting the same request repeatedly. + """ + key = self.result_key properties = self.extra_properties(eval_input) if self.extra_properties else {} properties["decision_model"] = { "deployment": self.runner.deployment, @@ -174,10 +192,6 @@ async def evaluate(self, eval_input: Dict[str, Any]) -> Dict[str, Any]: total_tokens=tokens["total_token_count"], model=tokens["model_id"], ) - usage_fields = { - f"{key}_{name}": properties[name] - for name in ("prompt_tokens", "completion_tokens", "total_tokens", "model") - } try: skip_reason = decision_skip_reason(response, STATUS_CUTOFF, self.error_target) properties["decision_model"]["status_completion_probability"] = decision_probability( @@ -187,7 +201,6 @@ async def evaluate(self, eval_input: Dict[str, Any]) -> Dict[str, Any]: if skip_reason: result = not_applicable_result(key, skip_reason, self.threshold) result[f"{key}_properties"] = properties - result.update(usage_fields) return result score, reason, details = self.scoring.interpret(response, key, self.error_target) except EvaluationException as exc: @@ -206,5 +219,4 @@ async def evaluate(self, eval_input: Dict[str, Any]) -> Dict[str, Any]: f"{key}_status": "completed", f"{key}_threshold": self.threshold, f"{key}_properties": properties, - **usage_fields, } diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py index e2e958e9111e..a1d43cabdfbf 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py @@ -59,7 +59,7 @@ def batch_client(): client._thread_pool.shutdown(wait=True) -def convert_batch(client, judge, tmp_path, rows=1): +def convert_batch(client, judge, tmp_path, rows=1, criteria=None): data = pd.DataFrame([{"query": "synthetic", "response": "synthetic", "ground_truth": "synthetic"}] * rows) judge_run = client.run(judge, data, evaluator_name="judge", column_mapping={"query": "${data.query}"}) f1_run = client.run( @@ -79,7 +79,7 @@ def convert_batch(client, judge, tmp_path, rows=1): row[key.replace("outputs.", f"outputs.{name}.", 1)] = value converted.append( _convert_single_row_to_aoai_format( - row, index, None, None, 0, CRITERIA, summaries, logging.getLogger("test") + row, index, None, None, 0, criteria or CRITERIA, summaries, logging.getLogger("test") ) ) path = tmp_path / "persisted.json" @@ -105,6 +105,134 @@ def strategy(runner): class TestDecisionBatchPersistence: + @pytest.mark.parametrize("members", [5, 6]) + @pytest.mark.parametrize("case", ["completed", "skipped", "mixed", "semantic_error"]) + @pytest.mark.parametrize("primary_first", [False, True]) + def test_shared_composite_request_is_counted_once_for_all_parsed_members( + self, batch_client, tmp_path, monkeypatch, members, case, primary_first + ): + calls = [] + keys = [f"member_{index}" for index in range(members)] + questions = { + key: {"type": "noul", "instructions": key} for member in keys for key in (member, f"{member}_status") + } + answers = {} + for index, key in enumerate(keys): + status = 0.1 if case == "skipped" or (case == "mixed" and index == 0) else 0.9 + score = ( + float("nan") + if case == "semantic_error" and index == members - 1 + else 0.1 if case == "mixed" and index == members - 1 else 0.9 + ) + answers[key] = {"type": "noul", "noul": score} + answers[f"{key}_status"] = {"type": "noul", "noul": status} + + class Response(HttpResponse): + def __init__(self, request): + super().__init__(request, None) + self.status_code = 200 + self.headers = {} + + def body(self): + return json.dumps( + { + "model": "decision-native", + "answers": answers, + "usage": { + "input_tokens": 120, + "output_tokens": 3, + }, + } + ).encode() + + class Transport(AsyncHttpTransport): + async def open(self): + pass + + async def close(self): + pass + + async def __aexit__(self, *_): + await self.close() + + async def send(self, request, **kwargs): + calls.append(json.loads(request.body)) + return Response(request) + + monkeypatch.setattr(_model_runners, "get_async_http_client", lambda: AsyncHttpPipeline(transport=Transport())) + + async def judge(query): + runner = DecisionModelRunner(CONFIG, max_retries=0) + shared = await runner(state={"query": query}, questions=questions, timeout=2) + result = {} + for key in keys: + member_strategy = DecisionEvaluationStrategy( + runner, + result_key=key, + error_target=ErrorTarget.EVALUATE, + questions={"status": questions[f"{key}_status"], key: questions[key]}, + scoring=BinaryScoring(0.75), + threshold=1, + timeout=2, + ) + member_response = { + **shared, + "answers": { + "status": shared["answers"][f"{key}_status"], + key: shared["answers"][key], + }, + } + result.update(member_strategy.parse(member_response, {"query": query})) + result.update( + { + "overall": None if case == "skipped" else float(case == "completed"), + "overall_status": "skipped" if case == "skipped" else "completed", + "overall_passed": None if case == "skipped" else case == "completed", + "overall_prompt_tokens": 120, + "overall_completion_tokens": 3, + "overall_total_tokens": 123, + "overall_model": shared["model"], + } + ) + return result + + criteria = { + **CRITERIA, + "judge": { + "metrics": ["overall", *keys] if primary_first else [*keys, "overall"], + "type": "azure_ai_evaluator", + "evaluator_name": "builtin.overall", + }, + } + _, rows = convert_batch(batch_client, judge, tmp_path, criteria=criteria) + assert len(calls) == 1 and len(calls[0]["questions"]) == members * 2 + for result in rows[0]["results"]: + if case == "semantic_error" and result["name"] == "judge": + assert result["status"] == "error" and result["score"] is None and result["passed"] is None + assert result["sample"]["error"]["blame"] == "SystemError" + assert result["properties"]["total_tokens"] == 123 + assert "native_answer" not in result["properties"]["decision_model"] + elif result["metric"] in keys: + assert "usage" not in (result.get("sample") or {}) + assert result["properties"]["total_tokens"] == 123 + assert len(result["properties"]["decision_model"]["rubric_hash"]) == 64 + if case == "skipped": + assert result["status"] == "skipped" and result["score"] is None and result["passed"] is None + assert sum("usage" in (result.get("sample") or {}) for result in rows[0]["results"]) == 1 + overall = next(result for result in rows[0]["results"] if result["metric"] == "overall") + assert overall["sample"]["usage"]["total_tokens"] == 123 + rollup = _calculate_aoai_evaluation_summary(rows, logging.getLogger("test"), criteria) + assert rollup["per_model_usage"] == [ + { + "model_name": "decision-native", + "invocation_count": 1, + "prompt_tokens": 120, + "completion_tokens": 3, + "total_tokens": 123, + "cached_tokens": 0, + } + ] + @pytest.mark.parametrize("status", [float("nan"), float("inf"), "bad"]) def test_semantic_failure_retains_measured_usage_and_hashes(self, batch_client, tmp_path, status): class Runner: diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py index 546bc5557182..cb9d3e6dd1a2 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py @@ -529,6 +529,18 @@ def strategy_with(payload, scoring, key="tool_call_accuracy", extra_properties=N class TestSharedDecisionStrategy: + @pytest.mark.parametrize("status", [0.1, 0.9]) + def test_parse_reuses_shared_response_without_http_or_usage_suffixes(self, status): + strategy = strategy_with(level_payload(4, status=status), LikertScoring(5)) + strategy.runner = Mock(deployment="Decision-1") + parsed = strategy.parse(level_payload(4, status=status), {}) + assert parsed["tool_call_accuracy_status"] == ("skipped" if status < 0.5 else "completed") + assert parsed["tool_call_accuracy_properties"]["total_tokens"] == 123 + assert not any( + key.endswith(("_prompt_tokens", "_completion_tokens", "_total_tokens", "_model")) for key in parsed + ) + strategy.runner.assert_not_called() + @pytest.mark.parametrize("probability, score", [(0.6, 1.0), (0.5999, 0.0)]) def test_binary_scoring_uses_its_own_cutoff(self, probability, score): strategy = strategy_with(completion_payload(probability), BinaryScoring(0.6), key="task_completion") From e31fd62646a865b0e90540722d5c61ed9e7d092e Mon Sep 17 00:00:00 2001 From: salma-elshafey Date: Tue, 13 Oct 2026 17:45:41 +0300 Subject: [PATCH 10/24] Avoid serializing uncapped Decision state question pairs Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e564c5c8-480e-48e7-a9e0-44450f774b5e --- .../_evaluators/_common/_model_runners.py | 30 ++++++++++--------- .../unittests/test_decision_model_support.py | 12 ++++++++ 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py index 086477a2562e..f1bb80c137f0 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py @@ -274,20 +274,22 @@ async def __call__(self, *, state: Any, questions: Dict[str, Dict[str, Any]], ti body = {"model": self._deployment, "state": state, "questions": questions} try: content = json.dumps(body, ensure_ascii=False, separators=(",", ":"), allow_nan=False).encode("utf-8") - state_question_bytes = max( - ( - len( - json.dumps( - {"state": state, "question": question}, - ensure_ascii=False, - separators=(",", ":"), - allow_nan=False, - ).encode("utf-8") - ) - for question in questions.values() - ), - default=0, - ) + state_question_bytes = 0 + if self._max_state_question_bytes is not None: + state_question_bytes = max( + ( + len( + json.dumps( + {"state": state, "question": question}, + ensure_ascii=False, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + ) + for question in questions.values() + ), + default=0, + ) except (ValueError, TypeError, OverflowError, UnicodeError): raise _failed( "Decision evidence must be finite JSON-serializable data.", ErrorBlame.USER_ERROR diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py index cb9d3e6dd1a2..ebf9ba6f2309 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py @@ -785,6 +785,18 @@ def test_request_budget_exact_boundary_and_no_send_on_overflow(self, client, fie assert not client.calls assert raised.value.blame == ErrorBlame.USER_ERROR + @pytest.mark.parametrize("request_cap", [None, 10000]) + def test_uncapped_state_question_pairs_are_not_serialized(self, client, request_cap): + client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) + with patch.object(_model_runners.json, "dumps", wraps=json.dumps) as serialize: + run(DecisionModelRunner(DECISION_CONFIG, max_request_bytes=request_cap)) + serialize.assert_called_once_with( + {"model": "Decision-1", "state": {"query": "q"}, "questions": QUESTIONS}, + ensure_ascii=False, + separators=(",", ":"), + allow_nan=False, + ) + def test_non_json_request_is_rejected_before_authentication_or_network(self, client): with pytest.raises(EvaluationException, match="JSON-serializable") as raised: run(DecisionModelRunner(DECISION_CONFIG), state={"value": float("nan")}) From f5d78f69cde9bc476ba6d6089bff358f41f8d80e Mon Sep 17 00:00:00 2001 From: salma-elshafey Date: Tue, 13 Oct 2026 18:20:12 +0300 Subject: [PATCH 11/24] Recognize asset quality composites in evaluation metadata Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e564c5c8-480e-48e7-a9e0-44450f774b5e --- .../azure-ai-evaluation/CHANGELOG.md | 4 +- .../azure/ai/evaluation/_constants.py | 19 ++++ .../ai/evaluation/_evaluate/_evaluate.py | 2 +- .../test_decision_batch_persistence.py | 99 ++++++++++++++++--- 4 files changed, 108 insertions(+), 16 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index f0b8ccd753ed..56232393b092 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -5,12 +5,14 @@ ### Features Added - Added shared model runners and evaluation strategies for asset evaluators using decision models in Microsoft Foundry. -- Decision evaluator results expose measured token usage and model fields in standard evaluation rows, including skipped evaluations, while retaining known metadata on failures. +- Decision evaluator results expose known final-response token usage and model fields in standard evaluation rows, including skipped evaluations, while retaining known metadata on failures. Usage does not include earlier retry attempts. ### Breaking Changes ### Bugs Fixed +- Output Quality and Tool Use Quality asset evaluators retain their aggregate and member metrics when used with `evaluate` without catalog metadata. + ### Other Changes ## 1.18.8 (2026-10-08) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_constants.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_constants.py index 3a379c29c8d7..b551805ee23b 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_constants.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_constants.py @@ -136,6 +136,23 @@ class _EvaluatorMetricMapping: "tool_call_success": ["tool_call_success"], "tool_selection": ["tool_selection"], "tool_output_utilization": ["tool_output_utilization"], + "output_quality": [ + "output_quality", + "fluency", + "coherence", + "intent_resolution", + "task_adherence", + "groundedness", + "task_completion", + ], + "tool_use_quality": [ + "tool_use_quality", + "tool_call_accuracy", + "tool_call_success", + "tool_input_accuracy", + "tool_output_utilization", + "tool_selection", + ], "task_navigation_efficiency": ["task_navigation_efficiency"], "text_similarity": ["similarity"], "string_check": ["string_check"], @@ -161,6 +178,7 @@ class _EvaluatorMetricMapping: "IndirectAttackEvaluator": "indirect_attack", "IntentResolutionEvaluator": "intent_resolution", "MeteorScoreEvaluator": "meteor_score", + "OutputQualityEvaluator": "output_quality", "ProtectedMaterialEvaluator": "protected_material", "QAEvaluator": "qa", "RelevanceEvaluator": "relevance", @@ -173,6 +191,7 @@ class _EvaluatorMetricMapping: "TaskAdherenceEvaluator": "task_adherence", "TaskCompletionEvaluator": "task_completion", "ToolCallAccuracyEvaluator": "tool_call_accuracy", + "ToolUseQualityEvaluator": "tool_use_quality", "UngroundedAttributesEvaluator": "ungrounded_attributes", "ViolenceEvaluator": "violence", } diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py index 67d891bfbfc6..d432380653e9 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py @@ -3386,7 +3386,7 @@ def _add_error_summaries( criteria_type = testing_criteria_metadata.get(criteria_name, {}).get("type", "azure_ai_evaluator") if decision_error_metadata is not None: evaluator_name = testing_criteria_metadata.get(criteria_name, {}).get("evaluator_name") - if isinstance(evaluator_name, str) and evaluator_name.startswith("builtin."): + if isinstance(evaluator_name, str): primary_metric = evaluator_name.removeprefix("builtin.") metrics = sorted(metrics, key=lambda metric: metric != primary_metric) decision_usage_added = False diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py index a1d43cabdfbf..a1efefea6838 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py @@ -17,7 +17,9 @@ from azure.ai.evaluation._evaluate._evaluate import ( _calculate_aoai_evaluation_summary, _convert_single_row_to_aoai_format, + _extract_testing_criteria_metadata, ) +from azure.ai.evaluation._evaluators._common._base_model_eval import ModelEvaluatorBase from azure.ai.evaluation._evaluators._common._decision_strategy import BinaryScoring, DecisionEvaluationStrategy from azure.ai.evaluation._evaluators._common._model_runners import DecisionModelRunner from azure.ai.evaluation._evaluators._common import _model_runners @@ -108,11 +110,33 @@ class TestDecisionBatchPersistence: @pytest.mark.parametrize("members", [5, 6]) @pytest.mark.parametrize("case", ["completed", "skipped", "mixed", "semantic_error"]) @pytest.mark.parametrize("primary_first", [False, True]) + @pytest.mark.parametrize("metadata_source", ["manual", "sdk_instance", "sdk_builtin"]) def test_shared_composite_request_is_counted_once_for_all_parsed_members( - self, batch_client, tmp_path, monkeypatch, members, case, primary_first + self, batch_client, tmp_path, monkeypatch, members, case, primary_first, metadata_source ): calls = [] keys = [f"member_{index}" for index in range(members)] + root = "overall" + if metadata_source != "manual": + if members == 6: + root = "output_quality" + keys = [ + "fluency", + "coherence", + "intent_resolution", + "task_adherence", + "groundedness", + "task_completion", + ] + else: + root = "tool_use_quality" + keys = [ + "tool_call_accuracy", + "tool_call_success", + "tool_input_accuracy", + "tool_output_utilization", + "tool_selection", + ] questions = { key: {"type": "noul", "instructions": key} for member in keys for key in (member, f"{member}_status") } @@ -161,8 +185,9 @@ async def send(self, request, **kwargs): monkeypatch.setattr(_model_runners, "get_async_http_client", lambda: AsyncHttpPipeline(transport=Transport())) - async def judge(query): - runner = DecisionModelRunner(CONFIG, max_retries=0) + async def judge(query, runner=None): + if runner is None: + runner = DecisionModelRunner(CONFIG, max_retries=0) shared = await runner(state={"query": query}, questions=questions, timeout=2) result = {} for key in keys: @@ -185,13 +210,16 @@ async def judge(query): result.update(member_strategy.parse(member_response, {"query": query})) result.update( { - "overall": None if case == "skipped" else float(case == "completed"), - "overall_status": "skipped" if case == "skipped" else "completed", - "overall_passed": None if case == "skipped" else case == "completed", - "overall_prompt_tokens": 120, - "overall_completion_tokens": 3, - "overall_total_tokens": 123, - "overall_model": shared["model"], + root: None if case == "skipped" else float(case == "completed"), + f"{root}_status": "skipped" if case == "skipped" else "completed", + f"{root}_passed": None if case == "skipped" else case == "completed", + f"{root}_result": ( + "not_applicable" if case == "skipped" else "pass" if case == "completed" else "fail" + ), + f"{root}_prompt_tokens": 120, + f"{root}_completion_tokens": 3, + f"{root}_total_tokens": 123, + f"{root}_model": shared["model"], } ) return result @@ -199,13 +227,47 @@ async def judge(query): criteria = { **CRITERIA, "judge": { - "metrics": ["overall", *keys] if primary_first else [*keys, "overall"], + "metrics": [root, *keys] if primary_first else [*keys, root], "type": "azure_ai_evaluator", - "evaluator_name": "builtin.overall", + "evaluator_name": f"builtin.{root}", }, } - _, rows = convert_batch(batch_client, judge, tmp_path, criteria=criteria) + evaluator = judge + if metadata_source != "manual": + + class OutputQualityEvaluator(ModelEvaluatorBase): + _supports_decision_models = True + _binary_decision = True + + def __call__(self, *, query): + return super().__call__(query=query) + + async def _do_eval(self, eval_input): + return await judge(eval_input["query"], self._decision_model_runner) + + class ToolUseQualityEvaluator(OutputQualityEvaluator): + pass + + evaluator_class = OutputQualityEvaluator if members == 6 else ToolUseQualityEvaluator + evaluator = evaluator_class(result_key=root, model_config=CONFIG, threshold=1, max_retries=0) + metadata = None + if metadata_source == "sdk_builtin": + metadata = { + "testing_criteria": [ + {"name": "judge", "type": "azure_ai_evaluator", "evaluator_name": f"builtin.{root}"} + ] + } + criteria = _extract_testing_criteria_metadata( + {"judge": evaluator, "f1": F1ScoreEvaluator()}, metadata, None, logging.getLogger("test"), None, None + ) + assert criteria["judge"]["metrics"] == [root, *keys] + assert criteria["judge"]["evaluator_name"] == (root if metadata is None else f"builtin.{root}") + assert criteria["judge"]["is_inverse"] is False + if not primary_first: + criteria["judge"]["metrics"] = [*keys, root] + _, rows = convert_batch(batch_client, evaluator, tmp_path, criteria=criteria) assert len(calls) == 1 and len(calls[0]["questions"]) == members * 2 + assert {result["metric"] for result in rows[0]["results"] if result["name"] == "judge"} == {root, *keys} for result in rows[0]["results"]: if case == "semantic_error" and result["name"] == "judge": assert result["status"] == "error" and result["score"] is None and result["passed"] is None @@ -219,9 +281,18 @@ async def judge(query): if case == "skipped": assert result["status"] == "skipped" and result["score"] is None and result["passed"] is None assert sum("usage" in (result.get("sample") or {}) for result in rows[0]["results"]) == 1 - overall = next(result for result in rows[0]["results"] if result["metric"] == "overall") + overall = next(result for result in rows[0]["results"] if result["metric"] == root) assert overall["sample"]["usage"]["total_tokens"] == 123 rollup = _calculate_aoai_evaluation_summary(rows, logging.getLogger("test"), criteria) + if metadata_source != "manual": + criteria_rollup = next( + item for item in rollup["per_testing_criteria_results"] if item["testing_criteria"] == "judge" + ) + expected = {"passed": 0, "failed": 0, "skipped": 0, "errored": 0} + expected[ + {"completed": "passed", "mixed": "failed", "skipped": "skipped", "semantic_error": "errored"}[case] + ] = 1 + assert criteria_rollup == {"testing_criteria": "judge", **expected} assert rollup["per_model_usage"] == [ { "model_name": "decision-native", From f4bde14ebcb201a22d1ceca05aa8b50ad2e6c8a1 Mon Sep 17 00:00:00 2001 From: salma-elshafey Date: Tue, 13 Oct 2026 19:37:17 +0300 Subject: [PATCH 12/24] Remove obsolete Decision execution policy guard Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e564c5c8-480e-48e7-a9e0-44450f774b5e --- .../ai/evaluation/_evaluators/_common/_base_model_eval.py | 7 ------- .../tests/unittests/test_decision_model_support.py | 4 ---- 2 files changed, 11 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py index 63345e0e4416..6351cdcdacdc 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py @@ -92,13 +92,6 @@ def __init__( _higher_is_better: bool = False, **kwargs, ) -> None: - if "decision_execution_policy" in kwargs: - raise EvaluationException( - message="Use explicit Decision execution controls instead of decision_execution_policy.", - blame=ErrorBlame.USER_ERROR, - category=ErrorCategory.INVALID_VALUE, - target=ErrorTarget.EVALUATE, - ) self._result_key = result_key self._is_reasoning_model = kwargs.get("is_reasoning_model", False) self._prompty_file = prompty_file diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py index ebf9ba6f2309..b01e1009731a 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py @@ -699,10 +699,6 @@ def test_base_forwards_controls_outside_strict_config(self): with pytest.raises(EvaluationException, match="Unsupported"): DecisionModelRunner({**DECISION_CONFIG, "max_retries": 0}) - def test_obsolete_base_policy_cannot_be_silently_ignored(self): - with pytest.raises(EvaluationException, match="explicit Decision"): - ProbeModelEvaluator(result_key="probe", model_config=DECISION_CONFIG, decision_execution_policy=object()) - @pytest.mark.parametrize("timeout", [0, -1, True, float("nan"), float("inf"), "30"]) def test_invalid_overall_timeouts_are_rejected(self, client, timeout): with pytest.raises(EvaluationException, match="positive finite"): From 45896c308f10ef9abe9b9db0378444dbaa407acf Mon Sep 17 00:00:00 2001 From: salma-elshafey Date: Tue, 13 Oct 2026 20:10:49 +0300 Subject: [PATCH 13/24] Carry overall Decision deadline into native transport Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e564c5c8-480e-48e7-a9e0-44450f774b5e --- .../_evaluators/_common/_base_model_eval.py | 2 + .../_evaluators/_common/_model_runners.py | 31 +++-- .../test_decision_default_transport.py | 117 ++++++++++++++++++ 3 files changed, 141 insertions(+), 9 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py index 6351cdcdacdc..92144dac079c 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py @@ -49,6 +49,8 @@ class ModelEvaluatorBase(EvaluatorBase[T]): file based on reasoning models. Defaults to False. :type is_reasoning_model: bool :keyword timeout: Optional overall Decision time limit, including authentication and retries. + The remaining budget also limits native transport connect/read timeouts. Awaiting is cancelled at the + deadline, but synchronous credential calls cannot be forcibly interrupted and may delay synchronous return. :paramtype timeout: Optional[float] :keyword timeout_seconds: Optional Decision connect/read timeout. :paramtype timeout_seconds: Optional[float] diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py index f1bb80c137f0..cec960092de3 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py @@ -13,6 +13,7 @@ import json import math import os +import time from typing import Any, Callable, Dict, List, Optional from azure.core.pipeline.policies import AsyncRedirectPolicy, AsyncRetryPolicy @@ -266,6 +267,7 @@ async def __call__(self, *, state: Any, questions: Dict[str, Dict[str, Any]], ti or timeout <= 0 ): raise _invalid_config("'timeout' must be a positive finite number.") + deadline = time.monotonic() + timeout for question in questions.values(): if question.get("type") not in ("noul", "score"): raise _invalid_config(f"Decision question type '{question.get('type')}' is not supported.") @@ -299,7 +301,9 @@ async def __call__(self, *, state: Any, questions: Dict[str, Dict[str, Any]], ti ): raise _failed("The decision request exceeds the configured byte budget.", ErrorBlame.USER_ERROR) try: - payload = await asyncio.wait_for(self._await_request(content), timeout=timeout) + payload = await asyncio.wait_for( + self._await_request(content, deadline), timeout=max(0, deadline - time.monotonic()) + ) except asyncio.TimeoutError: raise _failed("The decision model request timed out.") from None properties = self._response_properties(payload) @@ -324,12 +328,12 @@ def _cancelled(self) -> bool: raise _failed("The decision cancellation callback must return a boolean.", ErrorBlame.USER_ERROR) return result - async def _await_request(self, content: bytes) -> Any: + async def _await_request(self, content: bytes, deadline: float) -> Any: if self._cancelled(): raise _failed("The decision model request was cancelled.", ErrorBlame.USER_ERROR) if self._cancellation_requested is None: - return await self._post(content) - task = asyncio.create_task(self._post(content)) + return await self._post(content, deadline=deadline) + task = asyncio.create_task(self._post(content, deadline=deadline)) try: while not task.done(): if self._cancelled(): @@ -358,7 +362,7 @@ async def _get_default_token() -> str: async with AsyncAzureTokenProvider() as provider: return (await provider.get_token(_DECISION_TOKEN_SCOPE)).token - async def _post(self, content: bytes) -> Any: + async def _post(self, content: bytes, *, deadline: float) -> Any: headers = {"x-ms-useragent": self._user_agent, "Content-Type": "application/json"} if self._api_key: headers["api-key"] = self._api_key @@ -381,10 +385,17 @@ async def _post(self, content: bytes) -> Any: ), "redirect_policy": AsyncRedirectPolicy(permit_redirects=False), } - options: Dict[str, Any] = {"retry_on_methods": ["POST"]} - if self._timeout_seconds is not None: - # The default requests transport uses this scalar for both connect and read timeouts. - options["connection_timeout"] = self._timeout_seconds + remaining = deadline - time.monotonic() + if remaining <= 0: + raise _failed("The decision model request timed out.") + # Native retry policy carries this budget across attempts and clamps the scalar connect/read timeout. + options: Dict[str, Any] = { + "retry_on_methods": ["POST"], + "timeout": remaining, + "connection_timeout": ( + min(self._timeout_seconds, remaining) if self._timeout_seconds is not None else remaining + ), + } try: async with get_async_http_client().with_policies(**policies) as client: @@ -394,6 +405,8 @@ async def _post(self, content: bytes) -> Any: except EvaluationException: raise except Exception: # pylint: disable=broad-except + if time.monotonic() >= deadline: + raise _failed("The decision model request timed out.") from None raise _failed("The decision model request failed; check the endpoint and network access.") from None if self._max_response_bytes is not None and len(content) > self._max_response_bytes: if status == 200: diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_default_transport.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_default_transport.py index 24f723e8a807..175612bd6a6b 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_default_transport.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_default_transport.py @@ -15,6 +15,7 @@ from urllib3.util import connection from azure.ai.evaluation._evaluators._common._model_runners import DecisionModelRunner +from azure.ai.evaluation._evaluators._common._base_model_eval import ModelEvaluatorBase from azure.ai.evaluation._exceptions import ErrorBlame, EvaluationException from azure.ai.evaluation._http_utils import get_async_http_client from azure.core.pipeline.policies import AsyncRetryPolicy @@ -71,6 +72,122 @@ def invoke(endpoint, timeout=3, **controls): return asyncio.run(runner(state={"query": "é"}, questions=QUESTIONS, timeout=timeout)) +class SynchronousDecisionEvaluator(ModelEvaluatorBase): + _supports_decision_models = True + + def __call__(self, *, query): + return super().__call__(query=query) + + async def _do_eval(self, eval_input): + return await self._decision_model_runner(state=eval_input, questions=QUESTIONS, timeout=self._LLM_CALL_TIMEOUT) + + +@pytest.mark.parametrize("timeout,timeout_seconds", [(0.05, None), (0.2, 0.02), (0.05, 0.2)]) +def test_public_synchronous_overall_timeout_bounds_default_requests_worker( + localhost, monkeypatch, timeout, timeout_seconds +): + endpoint, calls, delay, _ = localhost + delay[0] = 0.65 + observed = [] + original = requests.Session.request + + def record(session, *args, **kwargs): + observed.append(kwargs["timeout"]) + return original(session, *args, **kwargs) + + monkeypatch.setattr(requests.Session, "request", record) + evaluator = SynchronousDecisionEvaluator( + result_key="status", + model_config={ + "type": "decision", + "azure_endpoint": endpoint, + "azure_deployment": "Decision-1", + "api_key": "local-test", + }, + timeout=timeout, + timeout_seconds=timeout_seconds, + max_retries=0, + ) + started = time.monotonic() + with pytest.raises(EvaluationException) as raised: + evaluator(query="synthetic") + elapsed = time.monotonic() - started + print(f"Entire public synchronous evaluator call: {elapsed:.3f}s") + assert elapsed < 0.4 + assert len(calls) == 1 + bound = min(timeout, timeout_seconds) if timeout_seconds is not None else timeout + assert len(observed) == 1 and 0 < observed[0] <= bound + 1e-8 + if timeout_seconds is None or timeout_seconds > timeout: + assert "timed out" in str(raised.value) + assert raised.value.properties == {"decision_model": {"deployment": "Decision-1"}} + + +def test_public_synchronous_authentication_consumes_transport_deadline(localhost, monkeypatch): + endpoint, calls, delay, _ = localhost + delay[0] = 0.65 + observed = [] + original = requests.Session.request + + def record(session, *args, **kwargs): + observed.append(kwargs["timeout"]) + return original(session, *args, **kwargs) + + class Credential: + async def get_token(self, *_): + await asyncio.sleep(0.08) + from azure.core.credentials import AccessToken + + return AccessToken("local-test", 9999999999) + + monkeypatch.setattr(requests.Session, "request", record) + evaluator = SynchronousDecisionEvaluator( + result_key="status", + model_config={"type": "decision", "azure_endpoint": endpoint, "azure_deployment": "Decision-1"}, + credential=Credential(), + timeout=0.15, + max_retries=0, + ) + started = time.monotonic() + with pytest.raises(EvaluationException, match="timed out"): + evaluator(query="synthetic") + assert time.monotonic() - started < 0.4 + assert len(calls) == 1 and len(observed) == 1 + assert 0 < observed[0] <= 0.075 + + +def test_public_synchronous_retries_and_backoff_share_remaining_deadline(localhost, monkeypatch): + endpoint, calls, delay, statuses = localhost + delay[0] = 0.04 + statuses[:] = [429, 429, 200] + observed = [] + original = requests.Session.request + + def record(session, *args, **kwargs): + observed.append(kwargs["timeout"]) + return original(session, *args, **kwargs) + + monkeypatch.setattr(requests.Session, "request", record) + evaluator = SynchronousDecisionEvaluator( + result_key="status", + model_config={ + "type": "decision", + "azure_endpoint": endpoint, + "azure_deployment": "Decision-1", + "api_key": "local-test", + }, + timeout=0.2, + max_retries=3, + ) + started = time.monotonic() + with pytest.raises(EvaluationException, match="timed out") as raised: + evaluator(query="synthetic") + assert time.monotonic() - started < 0.4 + assert len(calls) == len(observed) == 2 + assert 0 < observed[1] <= observed[0] - 0.03 + assert observed[0] <= 0.2 + 1e-8 + assert raised.value.properties == {"decision_model": {"deployment": "Decision-1"}} + + def test_default_transport_sends_policy_request_and_retries(localhost, monkeypatch): endpoint, calls, _, statuses = localhost statuses[:] = [429, 200] From 5f901dac59cf8731bc4e70ee978f423cc8abf692 Mon Sep 17 00:00:00 2001 From: salma-elshafey Date: Tue, 13 Oct 2026 20:16:53 +0300 Subject: [PATCH 14/24] Default Decision evaluations to a 120-second overall timeout Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e564c5c8-480e-48e7-a9e0-44450f774b5e --- .../_evaluators/_common/_base_model_eval.py | 5 ++++- .../tests/unittests/test_decision_model_support.py | 12 ++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py index 92144dac079c..2687c77ff98c 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py @@ -48,7 +48,8 @@ class ModelEvaluatorBase(EvaluatorBase[T]): :keyword is_reasoning_model: This parameter is in preview. If True, updates the config parameters in prompty file based on reasoning models. Defaults to False. :type is_reasoning_model: bool - :keyword timeout: Optional overall Decision time limit, including authentication and retries. + :keyword timeout: Optional overall Decision time limit, including authentication and retries. Defaults to + 120 seconds for Decision; an explicit value retains the existing evaluator timeout cap. The remaining budget also limits native transport connect/read timeouts. Awaiting is cancelled at the deadline, but synchronous credential calls cannot be forcibly interrupted and may delay synchronous return. :paramtype timeout: Optional[float] @@ -133,6 +134,8 @@ def __init__( target=ErrorTarget.EVALUATE, ) self._LLM_CALL_TIMEOUT = min(self._LLM_CALL_TIMEOUT, timeout) + else: + self._LLM_CALL_TIMEOUT = min(self._LLM_CALL_TIMEOUT, 120) self._model_runner: Union[DecisionModelRunner, PromptyModelRunner] = DecisionModelRunner( model_config, credential, diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py index b01e1009731a..2cc77c9196e3 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py @@ -664,6 +664,18 @@ def test_successful_properties_preserve_common_metadata_shape(self): class TestDecisionExecutionControls: + def test_overall_timeout_defaults_are_backend_specific(self): + decision = ProbeModelEvaluator(result_key="probe", model_config=DECISION_CONFIG) + with patch.object(_model_runners.AsyncPrompty, "load"): + chat = ProbeModelEvaluator(result_key="probe", prompty_file="probe.prompty", model_config=CHAT_CONFIG) + assert decision._LLM_CALL_TIMEOUT == 120 + assert chat._LLM_CALL_TIMEOUT == 600 + + @pytest.mark.parametrize("timeout,expected", [(5, 5), (300, 300), (900, 600)]) + def test_explicit_overall_timeout_retains_existing_cap(self, timeout, expected): + evaluator = ProbeModelEvaluator(result_key="probe", model_config=DECISION_CONFIG, timeout=timeout) + assert evaluator._LLM_CALL_TIMEOUT == expected + @pytest.mark.parametrize("name", ["execution_policy", "observer", "deadline_seconds"]) def test_obsolete_controls_are_not_accepted(self, name): with pytest.raises(TypeError, match=name): From 0df1cc9662616f064699ad5ff9360c8d70d49159 Mon Sep 17 00:00:00 2001 From: salma-elshafey Date: Wed, 14 Oct 2026 13:04:09 +0300 Subject: [PATCH 15/24] Enforce secure Decision endpoints and close owned credentials Validate Decision endpoint URLs before authentication and transport, allowing HTTP only for genuine loopback endpoints. Close default-provider owned credentials and environment resources across success, failure and cancellation while preserving caller ownership. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e564c5c8-480e-48e7-a9e0-44450f774b5e --- .../_evaluators/_common/_model_runners.py | 41 ++- .../_legacy/_common/_async_token_provider.py | 26 +- .../unittests/test_async_token_provider.py | 251 ++++++++++++++++++ .../unittests/test_decision_model_support.py | 63 ++++- 4 files changed, 373 insertions(+), 8 deletions(-) create mode 100644 sdk/evaluation/azure-ai-evaluation/tests/unittests/test_async_token_provider.py diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py index cec960092de3..d7d52132af37 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py @@ -10,11 +10,13 @@ import asyncio import inspect +import ipaddress import json import math import os import time from typing import Any, Callable, Dict, List, Optional +from urllib.parse import urlsplit from azure.core.pipeline.policies import AsyncRedirectPolicy, AsyncRetryPolicy @@ -161,7 +163,8 @@ class DecisionModelRunner: The model configuration must be tagged ``type="decision"`` and contain the Foundry resource endpoint (``azure_endpoint``) and the deployment name (``azure_deployment``). Authentication uses ``api_key`` when present, otherwise an Entra credential (from ``credential`` in the configuration or the constructor, or the - SDK default). ``api_version`` is accepted and ignored. + SDK default). The endpoint must use HTTPS; HTTP is permitted only for localhost or loopback IP literals. + Caller-supplied credentials remain caller-owned. ``api_version`` is accepted and ignored. ``probability_threshold`` (default 0.75) is only accepted when ``binary=True``. Execution controls are keyword-only and separate from model configuration. ``timeout_seconds`` bounds @@ -219,6 +222,42 @@ def __init__( endpoint = model_config.get("azure_endpoint") if not isinstance(endpoint, str) or not endpoint.strip(): raise _invalid_config("'azure_endpoint' must be a nonempty string.") + try: + parsed = urlsplit(endpoint) + hostname = parsed.hostname + port = parsed.port + loopback = hostname == "localhost" + if hostname and not loopback: + try: + loopback = ipaddress.ip_address(hostname).is_loopback + except ValueError: + labels = hostname.rstrip(".").encode("idna").decode("ascii").split(".") + if any( + not label + or not label[0].isalnum() + or not label[-1].isalnum() + or not all(character.isalnum() or character == "-" for character in label) + for label in labels + ): + raise ValueError + if ( + any(character.isspace() or ord(character) < 32 or ord(character) == 127 for character in endpoint) + or "\\" in endpoint + or not hostname + or "%" in hostname + or parsed.netloc.endswith(":") + or parsed.username is not None + or parsed.password is not None + or "?" in endpoint + or "#" in endpoint + or (port is not None and port == 0) + or (parsed.scheme != "https" and not (parsed.scheme == "http" and loopback)) + ): + raise ValueError + except ValueError: + raise _invalid_config( + "'azure_endpoint' must be a valid HTTPS URL; HTTP is allowed only for loopback." + ) from None deployment = model_config.get("azure_deployment") if not isinstance(deployment, str) or not deployment.strip(): raise _invalid_config("'azure_deployment' must be the decision model deployment name.") diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_async_token_provider.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_async_token_provider.py index 770b65ced1ca..fb870cb57a26 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_async_token_provider.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/_common/_async_token_provider.py @@ -14,7 +14,11 @@ class AsyncAzureTokenProvider(AsyncContextManager["AsyncAzureTokenProvider"]): """Asynchronous token provider for Azure services that supports non-default Azure clouds - (e.g. Azure China, Azure US Government, etc.).""" + (e.g. Azure China, Azure US Government, etc.). + + Closing the provider releases both its internally created synchronous credential and environment client. + Lazy token acquisition remains reusable until the provider is closed. + """ def __init__(self, *, base_url: Optional[str] = None, **kwargs: Any) -> None: """Initialize the AsyncAzureTokenProvider.""" @@ -22,11 +26,16 @@ def __init__(self, *, base_url: Optional[str] = None, **kwargs: Any) -> None: self._env_client: Optional[AzureEnvironmentClient] = AzureEnvironmentClient(base_url=base_url, **kwargs) async def close(self) -> None: - if self._env_client: - await self._env_client.close() - self._env_client = None - + credential, client = self._credential, self._env_client self._credential = None + self._env_client = None + try: + close_credential = getattr(credential, "close", None) + if close_credential is not None: + close_credential() + finally: + if client is not None: + await client.close() async def get_token( self, @@ -50,7 +59,12 @@ async def get_token( return self._credential.get_token(*scopes, claims=claims, tenant_id=tenant_id, enable_cae=enable_cae, **kwargs) async def __aenter__(self) -> "AsyncAzureTokenProvider": - self._credential = await self._initialize_async(self._env_client) + try: + if self._credential is None: + self._credential = await self._initialize_async(self._env_client) + except BaseException: + await self.close() + raise return self async def __aexit__( diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_async_token_provider.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_async_token_provider.py new file mode 100644 index 000000000000..730ba9c3f34f --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_async_token_provider.py @@ -0,0 +1,251 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- + +"""Owned default-provider resources and caller-owned Decision credentials.""" + +import asyncio +import json +import threading +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock + +import pytest + +from azure.ai.evaluation._legacy._common import _async_token_provider as providers +from azure.ai.evaluation._evaluators._common._model_runners import DecisionModelRunner +from azure.ai.evaluation._exceptions import EvaluationException + +from test_decision_model_support import DECISION_CONFIG, FakeClient, FakeResponse, QUESTIONS, level_payload + + +@pytest.fixture +def resources(monkeypatch): + client = Mock(close=AsyncMock()) + credential = Mock(get_token=Mock(return_value=SimpleNamespace(token="local-token"))) + monkeypatch.setattr(providers, "AzureEnvironmentClient", Mock(return_value=client)) + initialize = AsyncMock(return_value=credential) + monkeypatch.setattr(providers.AsyncAzureTokenProvider, "_initialize_async", initialize) + return client, credential, initialize + + +@pytest.mark.parametrize("failure", [None, RuntimeError, asyncio.CancelledError]) +def test_context_closes_owned_resources_after_token_outcome(resources, failure): + client, credential, _ = resources + if failure: + credential.get_token.side_effect = failure("token failed") + + async def request(): + async with providers.AsyncAzureTokenProvider() as provider: + return await provider.get_token("scope") + + if failure: + with pytest.raises(failure, match="token failed"): + asyncio.run(request()) + else: + assert asyncio.run(request()).token == "local-token" + credential.close.assert_called_once_with() + client.close.assert_awaited_once_with() + + +@pytest.mark.parametrize("failure", [RuntimeError, asyncio.CancelledError]) +def test_failed_context_entry_closes_environment_client(resources, failure): + client, credential, initialize = resources + initialize.side_effect = failure("initialization failed") + + async def request(): + async with providers.AsyncAzureTokenProvider(): + pytest.fail("Context entry must fail") + + with pytest.raises(failure, match="initialization failed"): + asyncio.run(request()) + client.close.assert_awaited_once_with() + credential.close.assert_not_called() + + +def test_lazy_provider_reuses_credential_and_close_is_idempotent(resources): + client, credential, initialize = resources + + async def request(): + provider = providers.AsyncAzureTokenProvider() + await provider.get_token("first") + await provider.get_token("second") + await provider.__aenter__() + await provider.close() + await provider.close() + + asyncio.run(request()) + initialize.assert_awaited_once() + assert credential.get_token.call_count == 2 + credential.close.assert_called_once_with() + client.close.assert_awaited_once_with() + + +def test_repeated_default_requests_close_each_owned_provider(monkeypatch): + from azure.ai.evaluation._evaluators._common import _model_runners + + clients = [Mock(close=AsyncMock()) for _ in range(3)] + credentials = [Mock(get_token=Mock(return_value=SimpleNamespace(token="local-token"))) for _ in range(3)] + monkeypatch.setattr(providers, "AzureEnvironmentClient", Mock(side_effect=clients)) + monkeypatch.setattr(providers.AsyncAzureTokenProvider, "_initialize_async", AsyncMock(side_effect=credentials)) + http = FakeClient(FakeResponse(body=json.dumps(level_payload(4)).encode())) + monkeypatch.setattr(_model_runners, "get_async_http_client", lambda: http) + config = {key: value for key, value in DECISION_CONFIG.items() if key != "api_key"} + + async def request(): + runner = DecisionModelRunner(config) + for _ in range(3): + await runner(state={}, questions=QUESTIONS, timeout=1) + + asyncio.run(request()) + assert len(http.calls) == 3 + for client, credential in zip(clients, credentials): + client.close.assert_awaited_once_with() + credential.close.assert_called_once_with() + + +@pytest.mark.parametrize("credential_fails,client_fails", [(True, False), (False, True), (True, True)]) +def test_close_attempts_both_resources_and_exposes_cleanup_failures(resources, credential_fails, client_fails): + client, credential, _ = resources + if credential_fails: + credential.close.side_effect = RuntimeError("credential close failed") + if client_fails: + client.close.side_effect = ValueError("environment close failed") + + async def request(): + provider = providers.AsyncAzureTokenProvider() + await provider.__aenter__() + try: + await provider.close() + finally: + await provider.close() + + expected = ValueError if client_fails else RuntimeError + with pytest.raises(expected) as raised: + asyncio.run(request()) + if credential_fails and client_fails: + assert isinstance(raised.value.__context__, RuntimeError) + credential.close.assert_called_once_with() + client.close.assert_awaited_once_with() + + +@pytest.mark.parametrize("outcome", ["success", "failure", "timeout", "cancelled"]) +def test_default_request_provider_ownership_and_worker_cleanup(resources, monkeypatch, outcome): + from azure.ai.evaluation._evaluators._common import _model_runners + + client, credential, _ = resources + http = FakeClient(FakeResponse(body=json.dumps(level_payload(4)).encode())) + monkeypatch.setattr(_model_runners, "get_async_http_client", lambda: http) + config = {key: value for key, value in DECISION_CONFIG.items() if key != "api_key"} + entered, release = threading.Event(), threading.Event() + if outcome == "failure": + credential.get_token.side_effect = RuntimeError("private token failure") + elif outcome in ("timeout", "cancelled"): + + def blocked_token(*_, **__): + entered.set() + assert release.wait(2) + return SimpleNamespace(token="local-token") + + credential.get_token.side_effect = blocked_token + + async def request(): + cancel = False + runner = DecisionModelRunner(config, cancellation_requested=lambda: cancel) + task = asyncio.create_task(runner(state={}, questions=QUESTIONS, timeout=0.1 if outcome == "timeout" else 2)) + try: + if outcome in ("timeout", "cancelled"): + while not entered.is_set(): + await asyncio.sleep(0.001) + cancel = outcome == "cancelled" + if outcome == "success": + await task + else: + with pytest.raises(EvaluationException): + await task + finally: + release.set() + + asyncio.run(request()) + credential.close.assert_called_once_with() + client.close.assert_awaited_once_with() + assert len(http.calls) == (1 if outcome == "success" else 0) + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("in_config", [False, True]) +@pytest.mark.parametrize("outcome", ["success", "failure", "cancelled"]) +def test_explicit_credentials_remain_caller_owned(monkeypatch, asynchronous, in_config, outcome): + from azure.ai.evaluation._evaluators._common import _model_runners + + http = FakeClient( + FakeResponse(body=json.dumps(level_payload(4)).encode()), delay=0.2 if outcome == "cancelled" else 0 + ) + monkeypatch.setattr(_model_runners, "get_async_http_client", lambda: http) + credential = Mock() + token_call = AsyncMock if asynchronous else Mock + credential.get_token = token_call(return_value=SimpleNamespace(token="caller-token")) + if outcome == "failure": + credential.get_token.side_effect = RuntimeError("private token failure") + config = {key: value for key, value in DECISION_CONFIG.items() if key != "api_key"} + if in_config: + config["credential"] = credential + + async def request(): + cancelled = asyncio.Event() + runner = DecisionModelRunner(config, None if in_config else credential, cancellation_requested=cancelled.is_set) + task = asyncio.create_task(runner(state={}, questions=QUESTIONS, timeout=1)) + if outcome == "cancelled": + while not http.calls: + await asyncio.sleep(0.001) + cancelled.set() + if outcome == "success": + await task + else: + with pytest.raises(EvaluationException): + await task + + asyncio.run(request()) + credential.close.assert_not_called() + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("termination", ["timeout", "cancelled"]) +def test_caller_credential_remains_open_when_authentication_is_interrupted(monkeypatch, asynchronous, termination): + from azure.ai.evaluation._evaluators._common import _model_runners + + http = FakeClient() + monkeypatch.setattr(_model_runners, "get_async_http_client", lambda: http) + entered, release = threading.Event(), threading.Event() + credential = Mock() + + def sync_token(*_): + entered.set() + assert release.wait(2) + return SimpleNamespace(token="caller-token") + + async def async_token(*_): + entered.set() + await asyncio.sleep(10) + + credential.get_token = AsyncMock(side_effect=async_token) if asynchronous else Mock(side_effect=sync_token) + config = {key: value for key, value in DECISION_CONFIG.items() if key != "api_key"} + + async def request(): + cancel = False + runner = DecisionModelRunner(config, credential, cancellation_requested=lambda: cancel) + task = asyncio.create_task( + runner(state={}, questions=QUESTIONS, timeout=0.1 if termination == "timeout" else 2) + ) + try: + while not entered.is_set(): + await asyncio.sleep(0.001) + cancel = termination == "cancelled" + with pytest.raises(EvaluationException): + await task + finally: + release.set() + + asyncio.run(request()) + credential.close.assert_not_called() + assert not http.calls diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py index 2cc77c9196e3..983f34140811 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py @@ -169,10 +169,69 @@ def test_invalid_probability_threshold(self, value): "not-a-url", ], ) - def test_endpoint_format_is_left_to_http_client(self, endpoint): + def test_endpoint_requires_secure_resource_url(self, endpoint): + if endpoint in ( + "https://example.services.ai.azure.com", + "https://example.services.ai.azure.com/", + "https://example.services.ai.azure.com/api/projects/proj", + ): + runner = DecisionModelRunner({**DECISION_CONFIG, "azure_endpoint": endpoint}) + assert runner._url == endpoint.rstrip("/") + "/providers/microsoft/v1/systemone" + else: + with pytest.raises(EvaluationException, match="HTTPS"): + DecisionModelRunner({**DECISION_CONFIG, "azure_endpoint": endpoint}) + + @pytest.mark.parametrize( + "endpoint", + ["http://localhost:8080", "http://LOCALHOST", "http://127.0.0.1:8080", "http://127.1.2.3", "http://[::1]:8080"], + ) + def test_http_is_allowed_for_loopback(self, endpoint): runner = DecisionModelRunner({**DECISION_CONFIG, "azure_endpoint": endpoint}) assert runner._url == endpoint.rstrip("/") + "/providers/microsoft/v1/systemone" + @pytest.mark.parametrize("authentication", ["key", "credential", "default"]) + @pytest.mark.parametrize( + "endpoint", + [ + "http://example.services.ai.azure.com", + "http://localhost.example.com", + "http://example-localhost", + "http://127.0.0.1.example.com", + "http://localhost@evil.example", + "http://evil.example@localhost", + "http://0.0.0.0", + "http://192.168.1.1", + "http://[::2]", + "http://local\thost", + "https://example.com\\@localhost", + "https://example.com?x=1", + "https://example.com#fragment", + "https://", + "https://example.com:bad", + "https://example.com:70000", + "https://example.com:", + "https://bad^host", + "https://bad..host", + "http://[::1%25anything]", + "https://example.com?", + "ftp://localhost", + "not-a-url", + ], + ) + def test_invalid_endpoint_is_rejected_before_authentication_or_transport(self, client, endpoint, authentication): + config = {**DECISION_CONFIG, "azure_endpoint": endpoint} + credential = Mock() + if authentication != "key": + config.pop("api_key") + with patch.object(DecisionModelRunner, "_get_default_token", new_callable=AsyncMock) as default_token: + with pytest.raises(EvaluationException, match="HTTPS") as raised: + DecisionModelRunner(config, credential if authentication == "credential" else None) + assert raised.value.blame == ErrorBlame.USER_ERROR + assert endpoint not in str(raised.value) + credential.get_token.assert_not_called() + default_token.assert_not_called() + assert not client.calls + def test_binary_probability_threshold(self): assert DecisionModelRunner(DECISION_CONFIG, binary=True).probability_threshold == 0.75 runner = DecisionModelRunner({**DECISION_CONFIG, "probability_threshold": 0.7}, binary=True) @@ -273,6 +332,7 @@ def get_token(*scopes, **kwargs): with patch.object(AsyncAzureTokenProvider, "_initialize_async", new=AsyncMock(return_value=credential)): run(DecisionModelRunner(config)) assert token_threads and all(token_thread != loop_thread for token_thread in token_threads) + credential.close.assert_called_once_with() assert client.calls[0]["headers"]["Authorization"] == "Bearer default-token" def test_default_credential_does_not_block_request_timeout(self, client): @@ -297,6 +357,7 @@ async def request(): with patch.object(AsyncAzureTokenProvider, "_initialize_async", new=AsyncMock(return_value=credential)): asyncio.run(request()) assert not client.calls + credential.close.assert_called_once_with() @pytest.mark.parametrize( "status, blame", From f15086a0817a87b6cd7986cb24a2da49f24ddbdc Mon Sep 17 00:00:00 2001 From: salma-elshafey Date: Wed, 14 Oct 2026 17:38:03 +0300 Subject: [PATCH 16/24] Make credential cancellation tests portable to Python 3.10 Assert cancellation by exception type without depending on asyncio.run preserving its message. Retain RuntimeError messages and all credential and environment cleanup assertions. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e564c5c8-480e-48e7-a9e0-44450f774b5e --- .../tests/unittests/test_async_token_provider.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_async_token_provider.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_async_token_provider.py index 730ba9c3f34f..8ad905ba14c6 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_async_token_provider.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_async_token_provider.py @@ -40,7 +40,7 @@ async def request(): return await provider.get_token("scope") if failure: - with pytest.raises(failure, match="token failed"): + with pytest.raises(failure, match="token failed" if failure is RuntimeError else None): asyncio.run(request()) else: assert asyncio.run(request()).token == "local-token" @@ -57,7 +57,7 @@ async def request(): async with providers.AsyncAzureTokenProvider(): pytest.fail("Context entry must fail") - with pytest.raises(failure, match="initialization failed"): + with pytest.raises(failure, match="initialization failed" if failure is RuntimeError else None): asyncio.run(request()) client.close.assert_awaited_once_with() credential.close.assert_not_called() From c7cba7a5dc35d980ed9c40d02fdd9c9094f59163 Mon Sep 17 00:00:00 2001 From: salma-elshafey Date: Fri, 16 Oct 2026 19:23:43 +0300 Subject: [PATCH 17/24] Address Decision evaluator review feedback Require explicit caller-owned authentication, validate probability sums, stop cancellable response reads, and preserve result accounting and metadata. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e564c5c8-480e-48e7-a9e0-44450f774b5e --- .../azure-ai-evaluation/CHANGELOG.md | 5 + .../_evaluators/_common/__init__.py | 9 + .../_evaluators/_common/_base_model_eval.py | 41 +- .../_evaluators/_common/_decision_strategy.py | 18 +- .../_evaluators/_common/_model_runners.py | 39 +- .../unittests/test_async_token_provider.py | 56 ++- .../test_decision_batch_persistence.py | 8 +- .../test_decision_default_transport.py | 220 ++++++++-- .../unittests/test_decision_model_support.py | 395 +++++++++++++++--- 9 files changed, 675 insertions(+), 116 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index c43cb19ba354..c7c673bb6580 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -9,13 +9,18 @@ ### Breaking Changes +- Decision evaluators now require an explicit `api_key` or a credential with a callable `get_token`; implicit default authentication is no longer selected. For Entra authentication, create and reuse a synchronous `azure.identity.DefaultAzureCredential`, pass it as `credential=`, and close it after all synchronous or batch evaluations finish. Async credentials must remain on the same caller-managed event loop. API-key precedence and Prompty authentication are unchanged. + ### Bugs Fixed +- Decision score answers now reject probability distributions whose sum differs from one by more than `1e-6`, reporting a system error instead of selecting a score from an invalid distribution. - Hardened local Prompty image resolution so only relative files within the Prompty directory are inlined, with opened-file validation to prevent path changes from redirecting reads and without exposing the local directory in fallback logs. - Output Quality and Tool Use Quality asset evaluators retain their aggregate and member metrics when used with `evaluate` without catalog metadata. ### Other Changes +- Decision requests use native async HTTP so overall timeout and cancellation stop response body reads. This path uses Python/OpenSSL TLS trust settings (for example, `SSL_CERT_FILE`), not the Requests-specific `REQUESTS_CA_BUNDLE`. Shared Prompty transport behavior is unchanged. + ## 1.18.9 (2026-10-12) ### Bugs Fixed diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/__init__.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/__init__.py index aa848b355e25..ed5605e44e22 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/__init__.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/__init__.py @@ -8,6 +8,9 @@ from ._base_rai_svc_eval import RaiServiceEvaluatorBase from ._base_multi_eval import MultiEvaluatorBase from ._conversation_input import hoist_messages_to_conversation +from ._model_runners import DecisionModelRunner, PromptyModelRunner +from ._decision_strategy import BinaryScoring, DecisionEvaluationStrategy, LikertScoring +from ._prompty_strategy import PromptyEvaluationStrategy __all__ = [ "EvaluatorBase", @@ -16,4 +19,10 @@ "RaiServiceEvaluatorBase", "MultiEvaluatorBase", "hoist_messages_to_conversation", + "DecisionModelRunner", + "PromptyModelRunner", + "DecisionEvaluationStrategy", + "PromptyEvaluationStrategy", + "BinaryScoring", + "LikertScoring", ] diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py index 2687c77ff98c..847195da453c 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py @@ -39,6 +39,11 @@ class ModelEvaluatorBase(EvaluatorBase[T]): only evaluators that set ``_supports_decision_models`` accept; any other configuration selects the Prompty runner and exposes its flow as ``_flow``. Interpreting a model's answer remains the evaluator's job. + Decision requires an explicit API key or a credential with a callable ``get_token``. For Entra authentication, + pass and reuse a synchronous ``azure.identity.DefaultAzureCredential`` as ``credential=`` for synchronous + and batch calls, closing it after all evaluations complete. Async credentials require the same caller-managed + event loop for all uses. The SDK never closes caller credentials. Prompty authentication is unchanged. + :param result_key: The key to use for the result of the evaluation. :type result_key: str :param prompty_file: The prompt path, required only for a Prompty runner. @@ -51,7 +56,8 @@ class ModelEvaluatorBase(EvaluatorBase[T]): :keyword timeout: Optional overall Decision time limit, including authentication and retries. Defaults to 120 seconds for Decision; an explicit value retains the existing evaluator timeout cap. The remaining budget also limits native transport connect/read timeouts. Awaiting is cancelled at the - deadline, but synchronous credential calls cannot be forcibly interrupted and may delay synchronous return. + deadline and the native async transport stops response body reads. Synchronous credential calls cannot be + forcibly interrupted and may delay synchronous return. :paramtype timeout: Optional[float] :keyword timeout_seconds: Optional Decision connect/read timeout. :paramtype timeout_seconds: Optional[float] @@ -196,6 +202,39 @@ def _decision_model_runner(self) -> DecisionModelRunner: raise RuntimeError("This evaluator is not configured with a decision model.") return runner + def _aggregate_results(self, per_turn_results: List[Dict[str, T]]) -> Dict[str, Any]: + """Aggregate scores normally, but sum independently known Decision usage across turns. + + A single shared model is promoted for attribution. Mixed or missing model identities remain per-turn + rather than attributing all usage to the last model. Prompty aggregation is unchanged. + """ + aggregated: Dict[str, Any] = super()._aggregate_results(per_turn_results) + if not self._uses_decision_model: + return aggregated + metrics = {metric for turn in per_turn_results for metric in turn} + for metric in metrics: + if metric.endswith(("_prompt_tokens", "_completion_tokens", "_total_tokens")): + values = (turn.get(metric) for turn in per_turn_results) + known = [ + value for value in values if isinstance(value, int) and not isinstance(value, bool) and value >= 0 + ] + if known: + aggregated[metric] = sum(known) + else: + aggregated.pop(metric, None) + elif metric.endswith("_model"): + models = [turn.get(metric) for turn in per_turn_results] + if ( + models + and isinstance(models[0], str) + and models[0].strip() + and all(model == models[0] for model in models) + ): + aggregated[metric] = models[0] + else: + aggregated.pop(metric, None) + return aggregated + def _get_binary_result(self, score: float) -> str: """Get the binary result based on the score. diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py index 2a7ef06094cd..6c31c72063c7 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py @@ -14,7 +14,7 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, TypeVar, Union from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException -from ._base_model_eval import ModelEvaluatorBase, not_applicable_result +from ._base_model_eval import not_applicable_result from ._model_runners import DecisionModelRunner, validate_noul_answer, validate_score_answer STATUS_CUTOFF = 0.5 @@ -83,7 +83,11 @@ def _digest(question: Dict[str, Any]) -> str: @dataclass(frozen=True) class BinaryScoring: - """Scores 1 when the answer's probability reaches the cutoff, otherwise 0.""" + """Scores 1 when the answer's probability reaches the cutoff, otherwise 0. + + Standalone results pass only at 1, regardless of the evaluator threshold. Composites can reapply their + configured member thresholds, so a binary member scoring 0 can pass when its composite threshold is 0. + """ probability_cutoff: float @@ -134,6 +138,11 @@ def passes(self, score: float, threshold: float) -> bool: class DecisionEvaluationStrategy: + """Interprets Decision answers using the fixed ``STATUS_CUTOFF`` of 0.5 for applicability. + + The evaluator's score threshold does not change the applicability cutoff. + """ + def __init__( self, runner: DecisionModelRunner, @@ -207,8 +216,9 @@ def parse(self, response: Dict[str, Any], eval_input: Dict[str, Any]) -> Dict[st exc.properties = properties raise properties["decision_model"].update(details) - properties["decision_model"]["native_answer"] = response["answers"][key] - properties.update(ModelEvaluatorBase._get_token_metadata(tokens)) + properties["decision_model"]["native_answer"] = { + name: value for name, value in response["answers"][key].items() if name != "legend" + } passed = self.scoring.passes(score, self.threshold) return { key: score, diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py index d7d52132af37..d0b97a9b0f2d 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py @@ -19,6 +19,7 @@ from urllib.parse import urlsplit from azure.core.pipeline.policies import AsyncRedirectPolicy, AsyncRetryPolicy +from azure.core.pipeline.transport import AioHttpTransport from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException from ..._common.utils import construct_prompty_model_config, validate_model_config @@ -52,6 +53,7 @@ def value(self) -> str: "probability_threshold", } _DEFAULT_PROBABILITY_THRESHOLD = 0.75 +_PROBABILITY_SUM_ABS_TOLERANCE = 1e-6 # The service adds api_version to every resolved model configuration; the decision endpoint does not use it. _IGNORED_CONFIG_KEYS = {"api_version"} @@ -138,6 +140,9 @@ def validate_noul_answer(answer: Dict[str, Any]) -> float: def validate_score_answer(answer: Dict[str, Any], levels: int) -> List[float]: """Return the level probabilities of a ``score`` answer after validating its fields. + Probabilities must sum to one within an absolute tolerance of ``1e-6`` for decimal rounding. + The native mean is checked for finiteness, not consistency with the distribution. + :param answer: The answer for a ``score`` question. :type answer: Dict[str, Any] :param levels: The number of levels in the question's scale. @@ -150,6 +155,8 @@ def validate_score_answer(answer: Dict[str, Any], levels: int) -> List[float]: if not isinstance(probabilities, dict) or set(probabilities) != {str(i) for i in range(levels)}: raise ValueError(f"a score answer requires probabilities for exactly {levels} levels.") distribution = [_probability(probabilities[str(i)]) for i in range(levels)] + if not math.isclose(math.fsum(distribution), 1.0, rel_tol=0.0, abs_tol=_PROBABILITY_SUM_ABS_TOLERANCE): + raise ValueError("score probabilities must sum to one within an absolute tolerance of 1e-6.") mean = answer.get("score") if isinstance(mean, bool) or not isinstance(mean, (int, float)) or not math.isfinite(mean): raise ValueError("the score mean must be finite.") @@ -162,13 +169,23 @@ class DecisionModelRunner: The model configuration must be tagged ``type="decision"`` and contain the Foundry resource endpoint (``azure_endpoint``) and the deployment name (``azure_deployment``). Authentication uses ``api_key`` when - present, otherwise an Entra credential (from ``credential`` in the configuration or the constructor, or the - SDK default). The endpoint must use HTTPS; HTTP is permitted only for localhost or loopback IP literals. + present, otherwise an explicitly supplied Entra credential (from ``credential`` in the configuration or the + constructor). No default credential is created for Decision. The endpoint must use HTTPS; + HTTP is permitted only for localhost or loopback IP literals. Caller-supplied credentials remain caller-owned. ``api_version`` is accepted and ignored. + Reuse a synchronous ``azure.identity.DefaultAzureCredential`` for public synchronous and batch calls, + and close it after all calls complete. Async credentials require a caller-managed event loop and must + only be reused on that loop; cross-loop reuse is not supported. ``probability_threshold`` (default 0.75) is only accepted when ``binary=True``. + ``azure_endpoint`` must be the account resource root, not a project or OpenAI API endpoint. + For example, use ``https://account.services.ai.azure.com`` rather than a URL ending in + ``/api/projects/project`` or ``/openai``. Any path supplied is retained when appending the provider operation. + Execution controls are keyword-only and separate from model configuration. ``timeout_seconds`` bounds - each connect/read; the call's ``timeout`` bounds authentication, retries and the entire operation. + each connect/read; the call's ``timeout`` limits authentication, retries and response body reading. + Decision requests use the native async HTTP transport so timeout and cancellation stop body reads. + An injected pipeline retains its own transport; that transport must support cancellation for the same guarantee. ``max_retries`` defaults to 3. ``cancellation_requested`` is polled every 50 ms, including retry waits; it stops awaiting synchronous credentials but cannot stop their worker threads. Byte limits measure compact unescaped UTF-8 JSON, with ``max_state_question_bytes`` applying to the largest individual pair. @@ -267,6 +284,8 @@ def __init__( # Like the Prompty path, an API key wins; a credential is only used when there is no key. credential = credential if credential is not None else model_config.get("credential") + if not api_key and not callable(getattr(credential, "get_token", None)): + raise _invalid_config("Decision models require an 'api_key' or a credential with a callable 'get_token'.") self._credential = None if api_key else credential threshold = model_config.get("probability_threshold", _DEFAULT_PROBABILITY_THRESHOLD) @@ -385,8 +404,6 @@ async def _await_request(self, content: bytes, deadline: float) -> Any: await asyncio.gather(task, return_exceptions=True) async def _get_token(self) -> str: - if self._credential is None: - return await asyncio.to_thread(lambda: asyncio.run(self._get_default_token())) if inspect.iscoroutinefunction(self._credential.get_token): return (await self._credential.get_token(_DECISION_TOKEN_SCOPE)).token token = await asyncio.to_thread(self._credential.get_token, _DECISION_TOKEN_SCOPE) @@ -394,13 +411,6 @@ async def _get_token(self) -> str: token = await token return token.token - @staticmethod - async def _get_default_token() -> str: - from azure.ai.evaluation._legacy._common._async_token_provider import AsyncAzureTokenProvider - - async with AsyncAzureTokenProvider() as provider: - return (await provider.get_token(_DECISION_TOKEN_SCOPE)).token - async def _post(self, content: bytes, *, deadline: float) -> Any: headers = {"x-ms-useragent": self._user_agent, "Content-Type": "application/json"} if self._api_key: @@ -435,9 +445,10 @@ async def _post(self, content: bytes, *, deadline: float) -> Any: min(self._timeout_seconds, remaining) if self._timeout_seconds is not None else remaining ), } - try: - async with get_async_http_client().with_policies(**policies) as client: + async with get_async_http_client( + transport=AioHttpTransport(read_timeout=options["connection_timeout"]) + ).with_policies(**policies) as client: response = await client.post(self._url, content=content, headers=headers, **options) status = response.status_code content = response.body() diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_async_token_provider.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_async_token_provider.py index 8ad905ba14c6..5986aa49a52c 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_async_token_provider.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_async_token_provider.py @@ -81,24 +81,18 @@ async def request(): client.close.assert_awaited_once_with() -def test_repeated_default_requests_close_each_owned_provider(monkeypatch): - from azure.ai.evaluation._evaluators._common import _model_runners - +def test_repeated_provider_contexts_close_each_owned_provider(monkeypatch): clients = [Mock(close=AsyncMock()) for _ in range(3)] credentials = [Mock(get_token=Mock(return_value=SimpleNamespace(token="local-token"))) for _ in range(3)] monkeypatch.setattr(providers, "AzureEnvironmentClient", Mock(side_effect=clients)) monkeypatch.setattr(providers.AsyncAzureTokenProvider, "_initialize_async", AsyncMock(side_effect=credentials)) - http = FakeClient(FakeResponse(body=json.dumps(level_payload(4)).encode())) - monkeypatch.setattr(_model_runners, "get_async_http_client", lambda: http) - config = {key: value for key, value in DECISION_CONFIG.items() if key != "api_key"} async def request(): - runner = DecisionModelRunner(config) for _ in range(3): - await runner(state={}, questions=QUESTIONS, timeout=1) + async with providers.AsyncAzureTokenProvider() as provider: + await provider.get_token("https://cognitiveservices.azure.com/.default") asyncio.run(request()) - assert len(http.calls) == 3 for client, credential in zip(clients, credentials): client.close.assert_awaited_once_with() credential.close.assert_called_once_with() @@ -130,12 +124,12 @@ async def request(): @pytest.mark.parametrize("outcome", ["success", "failure", "timeout", "cancelled"]) -def test_default_request_provider_ownership_and_worker_cleanup(resources, monkeypatch, outcome): +def test_explicit_request_credential_ownership_and_worker_cleanup(resources, monkeypatch, outcome): from azure.ai.evaluation._evaluators._common import _model_runners client, credential, _ = resources http = FakeClient(FakeResponse(body=json.dumps(level_payload(4)).encode())) - monkeypatch.setattr(_model_runners, "get_async_http_client", lambda: http) + monkeypatch.setattr(_model_runners, "get_async_http_client", lambda **_: http) config = {key: value for key, value in DECISION_CONFIG.items() if key != "api_key"} entered, release = threading.Event(), threading.Event() if outcome == "failure": @@ -151,7 +145,7 @@ def blocked_token(*_, **__): async def request(): cancel = False - runner = DecisionModelRunner(config, cancellation_requested=lambda: cancel) + runner = DecisionModelRunner(config, credential, cancellation_requested=lambda: cancel) task = asyncio.create_task(runner(state={}, questions=QUESTIONS, timeout=0.1 if outcome == "timeout" else 2)) try: if outcome in ("timeout", "cancelled"): @@ -167,11 +161,41 @@ async def request(): release.set() asyncio.run(request()) - credential.close.assert_called_once_with() - client.close.assert_awaited_once_with() + credential.close.assert_not_called() + client.close.assert_not_awaited() assert len(http.calls) == (1 if outcome == "success" else 0) +def test_repeated_decision_calls_reuse_caller_credential_without_creating_provider(resources, monkeypatch): + from azure.ai.evaluation._evaluators._common import _model_runners + + client, credential, initialize = resources + http = FakeClient(FakeResponse(body=json.dumps(level_payload(4)).encode())) + monkeypatch.setattr(_model_runners, "get_async_http_client", lambda **_: http) + config = {key: value for key, value in DECISION_CONFIG.items() if key != "api_key"} + runner = DecisionModelRunner(config, credential) + for _ in range(3): + asyncio.run(runner(state={}, questions=QUESTIONS, timeout=1)) + assert len(http.calls) == credential.get_token.call_count == 3 + initialize.assert_not_awaited() + credential.close.assert_not_called() + client.close.assert_not_awaited() + + +def test_prompty_still_constructs_default_provider_when_no_credential_is_supplied(resources): + from azure.ai.evaluation import FluencyEvaluator + from test_decision_model_support import CHAT_CONFIG + + client, credential, initialize = resources + evaluator = FluencyEvaluator({key: value for key, value in CHAT_CONFIG.items() if key != "api_key"}) + provider = evaluator._flow._token_credential + assert isinstance(provider, providers.AsyncAzureTokenProvider) + initialize.assert_not_awaited() + asyncio.run(provider.close()) + client.close.assert_awaited_once_with() + credential.close.assert_not_called() + + @pytest.mark.parametrize("asynchronous", [False, True]) @pytest.mark.parametrize("in_config", [False, True]) @pytest.mark.parametrize("outcome", ["success", "failure", "cancelled"]) @@ -181,7 +205,7 @@ def test_explicit_credentials_remain_caller_owned(monkeypatch, asynchronous, in_ http = FakeClient( FakeResponse(body=json.dumps(level_payload(4)).encode()), delay=0.2 if outcome == "cancelled" else 0 ) - monkeypatch.setattr(_model_runners, "get_async_http_client", lambda: http) + monkeypatch.setattr(_model_runners, "get_async_http_client", lambda **_: http) credential = Mock() token_call = AsyncMock if asynchronous else Mock credential.get_token = token_call(return_value=SimpleNamespace(token="caller-token")) @@ -215,7 +239,7 @@ def test_caller_credential_remains_open_when_authentication_is_interrupted(monke from azure.ai.evaluation._evaluators._common import _model_runners http = FakeClient() - monkeypatch.setattr(_model_runners, "get_async_http_client", lambda: http) + monkeypatch.setattr(_model_runners, "get_async_http_client", lambda **_: http) entered, release = threading.Event(), threading.Event() credential = Mock() diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py index a1efefea6838..f2dc3711757d 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py @@ -183,7 +183,9 @@ async def send(self, request, **kwargs): calls.append(json.loads(request.body)) return Response(request) - monkeypatch.setattr(_model_runners, "get_async_http_client", lambda: AsyncHttpPipeline(transport=Transport())) + monkeypatch.setattr( + _model_runners, "get_async_http_client", lambda **_: AsyncHttpPipeline(transport=Transport()) + ) async def judge(query, runner=None): if runner is None: @@ -414,7 +416,9 @@ async def send(self, request, **kwargs): cancelled = True return Response(request) - monkeypatch.setattr(_model_runners, "get_async_http_client", lambda: AsyncHttpPipeline(transport=Transport())) + monkeypatch.setattr( + _model_runners, "get_async_http_client", lambda **_: AsyncHttpPipeline(transport=Transport()) + ) async def judge(query): nonlocal cancelled diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_default_transport.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_default_transport.py index 175612bd6a6b..f1af01b2dfbb 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_default_transport.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_default_transport.py @@ -2,7 +2,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # --------------------------------------------------------- -"""Actual localhost HTTP coverage for the default requests-backed async transport.""" +"""Actual localhost HTTP coverage for Decision async HTTP and the unchanged shared requests transport.""" import asyncio import json @@ -11,14 +11,15 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import pytest +import aiohttp import requests -from urllib3.util import connection from azure.ai.evaluation._evaluators._common._model_runners import DecisionModelRunner from azure.ai.evaluation._evaluators._common._base_model_eval import ModelEvaluatorBase from azure.ai.evaluation._exceptions import ErrorBlame, EvaluationException from azure.ai.evaluation._http_utils import get_async_http_client from azure.core.pipeline.policies import AsyncRetryPolicy +from azure.core.pipeline.transport import AsyncioRequestsTransport pytestmark = pytest.mark.unittest QUESTIONS = {"status": {"type": "noul", "instructions": "Applicable?"}} @@ -83,19 +84,19 @@ async def _do_eval(self, eval_input): @pytest.mark.parametrize("timeout,timeout_seconds", [(0.05, None), (0.2, 0.02), (0.05, 0.2)]) -def test_public_synchronous_overall_timeout_bounds_default_requests_worker( +def test_public_synchronous_overall_timeout_bounds_native_async_transport( localhost, monkeypatch, timeout, timeout_seconds ): endpoint, calls, delay, _ = localhost delay[0] = 0.65 observed = [] - original = requests.Session.request + original = aiohttp.ClientSession.request def record(session, *args, **kwargs): - observed.append(kwargs["timeout"]) + observed.append(kwargs["timeout"].sock_connect) return original(session, *args, **kwargs) - monkeypatch.setattr(requests.Session, "request", record) + monkeypatch.setattr(aiohttp.ClientSession, "request", record) evaluator = SynchronousDecisionEvaluator( result_key="status", model_config={ @@ -126,10 +127,10 @@ def test_public_synchronous_authentication_consumes_transport_deadline(localhost endpoint, calls, delay, _ = localhost delay[0] = 0.65 observed = [] - original = requests.Session.request + original = aiohttp.ClientSession.request def record(session, *args, **kwargs): - observed.append(kwargs["timeout"]) + observed.append(kwargs["timeout"].sock_connect) return original(session, *args, **kwargs) class Credential: @@ -139,7 +140,7 @@ async def get_token(self, *_): return AccessToken("local-test", 9999999999) - monkeypatch.setattr(requests.Session, "request", record) + monkeypatch.setattr(aiohttp.ClientSession, "request", record) evaluator = SynchronousDecisionEvaluator( result_key="status", model_config={"type": "decision", "azure_endpoint": endpoint, "azure_deployment": "Decision-1"}, @@ -155,18 +156,21 @@ async def get_token(self, *_): assert 0 < observed[0] <= 0.075 -def test_public_synchronous_retries_and_backoff_share_remaining_deadline(localhost, monkeypatch): +@pytest.mark.parametrize("first_request_delay", [0, 0.16]) +def test_public_synchronous_retries_and_backoff_share_remaining_deadline(localhost, monkeypatch, first_request_delay): endpoint, calls, delay, statuses = localhost delay[0] = 0.04 statuses[:] = [429, 429, 200] observed = [] - original = requests.Session.request + original = aiohttp.ClientSession.request def record(session, *args, **kwargs): - observed.append(kwargs["timeout"]) + if not observed: + time.sleep(first_request_delay) + observed.append(kwargs["timeout"].sock_connect) return original(session, *args, **kwargs) - monkeypatch.setattr(requests.Session, "request", record) + monkeypatch.setattr(aiohttp.ClientSession, "request", record) evaluator = SynchronousDecisionEvaluator( result_key="status", model_config={ @@ -175,16 +179,17 @@ def record(session, *args, **kwargs): "azure_deployment": "Decision-1", "api_key": "local-test", }, - timeout=0.2, + # Allow scheduling headroom for two responses, but expire before the native 1.6s backoff. + timeout=0.8, max_retries=3, ) started = time.monotonic() with pytest.raises(EvaluationException, match="timed out") as raised: evaluator(query="synthetic") - assert time.monotonic() - started < 0.4 + assert time.monotonic() - started < 1.2 assert len(calls) == len(observed) == 2 assert 0 < observed[1] <= observed[0] - 0.03 - assert observed[0] <= 0.2 + 1e-8 + assert observed[0] <= 0.8 + 1e-8 assert raised.value.properties == {"decision_model": {"deployment": "Decision-1"}} @@ -192,25 +197,20 @@ def test_default_transport_sends_policy_request_and_retries(localhost, monkeypat endpoint, calls, _, statuses = localhost statuses[:] = [429, 200] observed_timeouts = [] - socket_connect_timeouts = [] - original = requests.Session.request - original_connect = connection.create_connection - - def record_connect(address, timeout=None, *args, **kwargs): - socket_connect_timeouts.append(timeout) - return original_connect(address, timeout, *args, **kwargs) + socket_read_timeouts = [] + original = aiohttp.ClientSession.request def record(session, *args, **kwargs): - observed_timeouts.append(kwargs["timeout"]) + observed_timeouts.append(kwargs["timeout"].sock_connect) + socket_read_timeouts.append(kwargs["timeout"].sock_read) assert "read_timeout" not in kwargs and "connection_timeout" not in kwargs return original(session, *args, **kwargs) - monkeypatch.setattr(requests.Session, "request", record) - monkeypatch.setattr(connection, "create_connection", record_connect) + monkeypatch.setattr(aiohttp.ClientSession, "request", record) result = invoke(endpoint, timeout_seconds=0.5, max_retries=1) assert result["usage"] == {"input_tokens": 12, "output_tokens": 4} assert observed_timeouts == [0.5, 0.5] - assert socket_connect_timeouts == [0.5, 0.5] + assert socket_read_timeouts == [0.5, 0.5] assert len(calls) == 2 and calls[0]["body"] == calls[1]["body"] assert calls[0]["path"] == "/providers/microsoft/v1/systemone" assert calls[0]["content_type"] == "application/json" @@ -267,6 +267,55 @@ async def request(): assert observed == [0.7] +def test_decision_preserves_factory_injected_requests_transport(localhost, monkeypatch): + from azure.ai.evaluation._evaluators._common import _model_runners + + endpoint, calls, _, _ = localhost + observed = [] + original = requests.Session.request + + def record(session, *args, **kwargs): + observed.append(kwargs["timeout"]) + assert "read_timeout" not in kwargs + return original(session, *args, **kwargs) + + monkeypatch.setattr(requests.Session, "request", record) + monkeypatch.setattr( + _model_runners, + "get_async_http_client", + lambda **_: get_async_http_client(transport=AsyncioRequestsTransport()), + ) + assert invoke(endpoint, timeout_seconds=0.7)["usage"] == {"input_tokens": 12, "output_tokens": 4} + assert observed == [0.7] + assert len(calls) == 1 + + +def test_repeated_public_calls_close_async_sessions_on_their_own_loops(localhost, monkeypatch): + endpoint, calls, _, _ = localhost + closed = [] + original_close = aiohttp.ClientSession.close + + async def close(session): + await original_close(session) + closed.append((session, asyncio.get_running_loop())) + + monkeypatch.setattr(aiohttp.ClientSession, "close", close) + evaluator = SynchronousDecisionEvaluator( + result_key="status", + model_config={ + "type": "decision", + "azure_endpoint": endpoint, + "azure_deployment": "Decision-1", + "api_key": "local-test", + }, + ) + for _ in range(2): + assert evaluator(query="synthetic")["usage"] == {"input_tokens": 12, "output_tokens": 4} + assert len(calls) == len(closed) == 2 + assert all(session.closed and loop.is_closed() for session, loop in closed) + assert closed[0][1] is not closed[1][1] + + def test_default_transport_cancellation_interrupts_awaiting_a_real_request(localhost): endpoint, calls, delay, _ = localhost delay[0] = 0.5 @@ -292,3 +341,120 @@ async def request(): asyncio.run(request()) assert len(calls) == 1 + + +@pytest.fixture +def trickling_endpoint(): + sent = [] + disconnected = threading.Event() + disconnected_at = [] + payload = json.dumps( + { + "model": "native-decision", + "answers": {"status": {"type": "noul", "noul": 0.9}}, + "usage": {"input_tokens": 12, "output_tokens": 4}, + "padding": "x" * 160, + } + ).encode() + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *_): + pass + + def do_POST(self): + self.rfile.read(int(self.headers["Content-Length"])) + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + for offset in range(0, len(payload), 4): + try: + self.wfile.write(payload[offset : offset + 4]) + self.wfile.flush() + sent.append(len(payload[offset : offset + 4])) + except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError): + disconnected_at.append(time.monotonic()) + disconnected.set() + break + time.sleep(0.03) + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}", sent, disconnected, disconnected_at, len(payload) + finally: + server.shutdown() + server.server_close() + thread.join() + + +@pytest.mark.parametrize("batch", [False, True], ids=["public_sync", "batch"]) +@pytest.mark.parametrize("cancel", [False, True], ids=["deadline", "cancellation"]) +def test_trickling_body_is_stopped_by_overall_deadline_or_cancellation(trickling_endpoint, monkeypatch, batch, cancel): + import pandas as pd + + from azure.ai.evaluation._evaluate._batch_run import RunSubmitterClient + + endpoint, sent, disconnected, disconnected_at, length = trickling_endpoint + closed_sessions = [] + original_close = aiohttp.ClientSession.close + + async def close(session): + await original_close(session) + closed_sessions.append(session) + + monkeypatch.setattr(aiohttp.ClientSession, "close", close) + cancelled = threading.Event() + evaluator = SynchronousDecisionEvaluator( + result_key="status", + model_config={ + "type": "decision", + "azure_endpoint": endpoint, + "azure_deployment": "Decision-1", + "api_key": "local-test", + }, + timeout=3 if cancel else 0.2, + cancellation_requested=cancelled.is_set if cancel else None, + max_retries=0, + ) + client = RunSubmitterClient() if batch else None + timer = threading.Timer(0.2, cancelled.set) if cancel else None + try: + if timer: + timer.start() + started = time.monotonic() + if client: + run = client.run( + evaluator, + pd.DataFrame([{"query": "synthetic"}]), + evaluator_name="judge", + column_mapping={"query": "${data.query}"}, + ) + summary = client.get_run_summary(run) + assert summary["failed_lines"] == 1 + assert summary["completed_lines"] == 0 + else: + with pytest.raises(EvaluationException, match="cancelled" if cancel else "timed out") as raised: + evaluator(query="synthetic") + assert raised.value.properties == {"decision_model": {"deployment": "Decision-1"}} + elapsed = time.monotonic() - started + # The existing batch engine polls run completion every second; the HTTP read must stop sooner. + assert elapsed < (1.5 if batch else 0.8) + assert 0 < sum(sent) < length + assert disconnected.wait(0.5), "The server must observe the HTTP body read stopping." + assert disconnected_at[0] - started < 0.8 + assert len(closed_sessions) == 1 and closed_sessions[0].closed + stopped_at = sum(sent) + time.sleep(0.09) + assert sum(sent) == stopped_at < length + print( + f"batch={batch} cancel={cancel}: returned={elapsed:.3f}s disconnected=" + f"{disconnected_at[0] - started:.3f}s bytes={stopped_at}/{length}" + ) + finally: + if timer: + timer.cancel() + timer.join() + if client: + client._thread_pool.shutdown(wait=True) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py index 983f34140811..879c6f62b301 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py @@ -7,17 +7,19 @@ import asyncio import hashlib import json +import math import threading from concurrent.futures import ThreadPoolExecutor from types import SimpleNamespace from unittest.mock import AsyncMock, Mock, patch import pytest +from openai.types.chat import ChatCompletion from azure.core.pipeline.policies import AsyncRedirectPolicy, AsyncRetryPolicy from azure.core.pipeline.transport import AsyncHttpTransport, HttpResponse from azure.core.exceptions import ServiceRequestError -from azure.ai.evaluation import FluencyEvaluator +from azure.ai.evaluation import CoherenceEvaluator, FluencyEvaluator from azure.ai.evaluation._evaluators._common import EvaluatorBase, ModelEvaluatorBase, PromptyEvaluatorBase from azure.ai.evaluation._evaluators._common import _model_runners from azure.ai.evaluation._evaluators._common._decision_strategy import ( @@ -31,7 +33,7 @@ validate_noul_answer, validate_score_answer, ) -from azure.ai.evaluation._exceptions import ErrorBlame, ErrorTarget, EvaluationException +from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException pytestmark = pytest.mark.unittest @@ -114,7 +116,7 @@ async def post(self, url, content=None, headers=None, **options): @pytest.fixture def client(monkeypatch): fake = FakeClient() - monkeypatch.setattr(_model_runners, "get_async_http_client", lambda: fake) + monkeypatch.setattr(_model_runners, "get_async_http_client", lambda **_: fake) return fake @@ -223,13 +225,11 @@ def test_invalid_endpoint_is_rejected_before_authentication_or_transport(self, c credential = Mock() if authentication != "key": config.pop("api_key") - with patch.object(DecisionModelRunner, "_get_default_token", new_callable=AsyncMock) as default_token: - with pytest.raises(EvaluationException, match="HTTPS") as raised: - DecisionModelRunner(config, credential if authentication == "credential" else None) + with pytest.raises(EvaluationException, match="HTTPS") as raised: + DecisionModelRunner(config, credential if authentication == "credential" else None) assert raised.value.blame == ErrorBlame.USER_ERROR assert endpoint not in str(raised.value) credential.get_token.assert_not_called() - default_token.assert_not_called() assert not client.calls def test_binary_probability_threshold(self): @@ -238,9 +238,38 @@ def test_binary_probability_threshold(self): assert runner.probability_threshold == 0.7 def test_api_key_wins_over_credential(self): - runner = DecisionModelRunner(DECISION_CONFIG, credential=Mock()) + class IgnoredCredential: + @property + def get_token(self): + raise AssertionError("Ignored credential must not be inspected.") + + runner = DecisionModelRunner(DECISION_CONFIG, credential=IgnoredCredential()) assert runner._credential is None and runner._api_key == DECISION_CONFIG["api_key"] + @pytest.mark.parametrize( + "credential", [None, object(), "token-secret", {"get_token": "secret"}, SimpleNamespace(get_token=3)] + ) + @pytest.mark.parametrize("in_config", [False, True]) + def test_missing_or_invalid_explicit_auth_is_rejected_at_construction(self, client, credential, in_config): + config = {key: value for key, value in DECISION_CONFIG.items() if key != "api_key"} + if in_config: + config["credential"] = credential + with pytest.raises(EvaluationException, match="callable 'get_token'") as raised: + DecisionModelRunner(config, None if in_config else credential) + assert raised.value.blame == ErrorBlame.USER_ERROR + assert raised.value.category == ErrorCategory.INVALID_VALUE + assert raised.value.target == ErrorTarget.EVALUATE + assert "token-secret" not in str(raised.value) + assert not client.calls + + def test_constructor_credential_takes_precedence_without_identity_fallback(self): + config = {key: value for key, value in DECISION_CONFIG.items() if key != "api_key"} + configured = Mock() + supplied = Mock() + assert DecisionModelRunner({**config, "credential": configured}, supplied)._credential is supplied + with pytest.raises(EvaluationException, match="callable 'get_token'"): + DecisionModelRunner({**config, "credential": configured}, object()) + class TestDecisionModelRunnerRequests: def test_request_shape_with_api_key(self, client): @@ -311,14 +340,12 @@ async def send(self, request, **kwargs): return Response(request, status if self.calls == 1 else 200) transport = Transport() - monkeypatch.setattr(_model_runners, "get_async_http_client", lambda: AsyncHttpPipeline(transport=transport)) + monkeypatch.setattr(_model_runners, "get_async_http_client", lambda **_: AsyncHttpPipeline(transport=transport)) result = run(DecisionModelRunner(DECISION_CONFIG)) assert transport.calls == 2 assert result["model"] == "microsoft-decision-1" - def test_default_credential_token_acquisition_runs_off_event_loop(self, client): - from azure.ai.evaluation._legacy._common._async_token_provider import AsyncAzureTokenProvider - + def test_explicit_sync_credential_token_acquisition_runs_off_event_loop(self, client): client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) loop_thread = threading.get_ident() token_threads = [] @@ -329,15 +356,12 @@ def get_token(*scopes, **kwargs): credential = Mock(get_token=get_token) config = {key: value for key, value in DECISION_CONFIG.items() if key != "api_key"} - with patch.object(AsyncAzureTokenProvider, "_initialize_async", new=AsyncMock(return_value=credential)): - run(DecisionModelRunner(config)) + run(DecisionModelRunner(config, credential)) assert token_threads and all(token_thread != loop_thread for token_thread in token_threads) - credential.close.assert_called_once_with() + credential.close.assert_not_called() assert client.calls[0]["headers"]["Authorization"] == "Bearer default-token" - def test_default_credential_does_not_block_request_timeout(self, client): - from azure.ai.evaluation._legacy._common._async_token_provider import AsyncAzureTokenProvider - + def test_explicit_sync_credential_does_not_block_request_timeout(self, client): release = threading.Event() def get_token(*scopes, **kwargs): @@ -350,14 +374,13 @@ def get_token(*scopes, **kwargs): async def request(): try: with pytest.raises(EvaluationException, match="timed out"): - await DecisionModelRunner(config)(state={}, questions=QUESTIONS, timeout=0.02) + await DecisionModelRunner(config, credential)(state={}, questions=QUESTIONS, timeout=0.02) finally: release.set() - with patch.object(AsyncAzureTokenProvider, "_initialize_async", new=AsyncMock(return_value=credential)): - asyncio.run(request()) + asyncio.run(request()) assert not client.calls - credential.close.assert_called_once_with() + credential.close.assert_not_called() @pytest.mark.parametrize( "status, blame", @@ -382,7 +405,7 @@ class Broken(FakeClient): async def post(self, *_, **__): raise ConnectionError("https://example.services.ai.azure.com test-key-value") - monkeypatch.setattr(_model_runners, "get_async_http_client", lambda: Broken()) + monkeypatch.setattr(_model_runners, "get_async_http_client", lambda **_: Broken()) with pytest.raises(EvaluationException) as raised: run(DecisionModelRunner(DECISION_CONFIG)) assert "test-key-value" not in str(raised.value) and "example.services" not in str(raised.value) @@ -440,12 +463,36 @@ def test_noul(self): with pytest.raises(ValueError): validate_noul_answer({"type": "noul", "noul": bad}) - def test_score_fields_do_not_require_consistency_checks(self): - levels = [0.1, 0.2, 0.3, 0.4, 0.5] + def test_score_mean_does_not_require_distribution_consistency(self): + levels = [0.1, 0.2, 0.3, 0.2, 0.2] answer = score_answer(levels) answer["score"] = 0 assert validate_score_answer(answer, 5) == levels + def test_score_accepts_rounded_distribution_without_normalizing(self): + levels = [0.333333, 0.333333, 0.333334] + assert validate_score_answer(score_answer(levels), 3) == levels + levels = [0.3333333] * 3 + assert validate_score_answer(score_answer(levels), 3) == levels + + @pytest.mark.parametrize("levels", [[0] * 5, [0.1] * 5, [0.3] * 5]) + def test_score_rejects_nonunit_distribution(self, levels): + with pytest.raises(ValueError, match="sum to one"): + validate_score_answer(score_answer(levels), 5) + + @pytest.mark.parametrize("direction", [-1, 1]) + @pytest.mark.parametrize("inside", [False, True]) + def test_score_sum_tolerance_boundary(self, direction, inside): + boundary = 1 + direction * 1e-6 + total = math.nextafter(boundary, 1.0 if inside else direction * math.inf) + levels = [0.5, total - 0.5] + assert math.fsum(levels) == total + if inside: + assert validate_score_answer(score_answer(levels), 2) == levels + else: + with pytest.raises(ValueError, match="sum to one"): + validate_score_answer(score_answer(levels), 2) + @pytest.mark.parametrize( "mutate", [ @@ -472,6 +519,21 @@ async def _do_eval(self, eval_input): class TestSharedBaseSelection: + def test_common_exports_match_existing_private_imports(self): + from azure.ai.evaluation._evaluators import _common + from azure.ai.evaluation._evaluators._common._prompty_strategy import PromptyEvaluationStrategy + + for name, implementation in ( + ("DecisionModelRunner", DecisionModelRunner), + ("PromptyModelRunner", PromptyModelRunner), + ("DecisionEvaluationStrategy", DecisionEvaluationStrategy), + ("PromptyEvaluationStrategy", PromptyEvaluationStrategy), + ("BinaryScoring", BinaryScoring), + ("LikertScoring", LikertScoring), + ): + assert getattr(_common, name) is implementation + assert name in _common.__all__ + def test_hierarchy(self): assert issubclass(ModelEvaluatorBase, EvaluatorBase) assert issubclass(PromptyEvaluatorBase, ModelEvaluatorBase) @@ -569,6 +631,88 @@ def test_inherited_helpers_remain_available_to_prompty_evaluators(self): assert evaluator._get_token_metadata({"input_token_count": 3})["prompt_tokens"] == 3 assert evaluator._return_not_applicable_result("why", 3)["fluency_status"] == "skipped" + def test_bundled_fluency_public_call_uses_legacy_prompty_parsing(self): + flow = AsyncMock( + return_value={ + "llm_output": "Clear prose4", + "input_token_count": 12, + "output_token_count": 3, + "total_token_count": 15, + "model_id": "chat-model", + } + ) + with patch.object(_model_runners.AsyncPrompty, "load", return_value=flow): + evaluator = FluencyEvaluator(CHAT_CONFIG) + result = evaluator(response="A clear sentence.") + assert result["fluency"] == 4 + assert result["fluency_reason"] == "Clear prose" + assert result["fluency_result"] == "pass" + assert result["fluency_properties"]["model"] == "chat-model" + assert result["fluency_properties"]["prompt_tokens"] == 12 + flow.assert_awaited_once_with(timeout=600, response="A clear sentence.") + + @pytest.mark.parametrize( + "evaluator_type,metric", [(FluencyEvaluator, "fluency"), (CoherenceEvaluator, "coherence")] + ) + @pytest.mark.parametrize("multi_turn", [False, True]) + def test_bundled_prompty_public_call_loads_real_template(self, evaluator_type, metric, multi_turn): + completion = ChatCompletion( + id="offline-completion", + created=0, + object="chat.completion", + model="offline-chat-model", + choices=[ + { + "index": 0, + "finish_reason": "stop", + "message": { + "role": "assistant", + "content": json.dumps({"reason": "Clear prose", "score": 4, "status": "completed"}), + }, + } + ], + usage={"prompt_tokens": 12, "completion_tokens": 3, "total_tokens": 15}, + ) + + async def send(*, api_client, params, timeout): + await api_client.close() + assert timeout == 600 + assert params["response_format"] == {"type": "json_object"} + assert params["messages"] + assert any("A clear sentence." in str(message["content"]) for message in params["messages"]) + return completion + + with patch.object(_model_runners.AsyncPrompty, "_send_with_retries", side_effect=send) as request: + evaluator = evaluator_type(CHAT_CONFIG) + assert evaluator._flow.path.is_file() + assert evaluator._flow.path.name == f"{metric}.prompty" + if multi_turn: + result = evaluator( + conversation={ + "messages": [ + {"role": "user", "content": "Explain clearly."}, + {"role": "assistant", "content": "A clear sentence."}, + {"role": "user", "content": "Explain again."}, + {"role": "assistant", "content": "A clear sentence."}, + ] + } + ) + else: + inputs = {"response": "A clear sentence."} + if metric == "coherence": + inputs["query"] = "Explain clearly." + result = evaluator(**inputs) + assert result[metric] == 4 + assert request.await_count == (2 if multi_turn else 1) + if multi_turn: + assert result["evaluation_per_turn"][metric] == [4, 4] + assert result["evaluation_per_turn"][f"{metric}_reason"] == ["Clear prose"] * 2 + else: + assert result[f"{metric}_reason"] == "Clear prose" + assert result[f"{metric}_result"] == "pass" + assert result[f"{metric}_properties"]["model"] == "offline-chat-model" + assert result[f"{metric}_properties"]["prompt_tokens"] == 12 + def completion_payload(probability, status=0.95): return decision_payload(status, {"task_completion": {"type": "noul", "noul": probability}}) @@ -607,6 +751,14 @@ def test_binary_scoring_uses_its_own_cutoff(self, probability, score): strategy = strategy_with(completion_payload(probability), BinaryScoring(0.6), key="task_completion") assert asyncio.run(strategy.evaluate({}))["task_completion"] == score + def test_standalone_binary_result_does_not_use_score_threshold(self): + strategy = strategy_with(completion_payload(0.1), BinaryScoring(0.6), key="task_completion") + strategy.threshold = 0 + result = asyncio.run(strategy.evaluate({})) + assert result["task_completion"] == 0 + assert result["task_completion_threshold"] == 0 + assert result["task_completion_passed"] is False + @pytest.mark.parametrize("minimum, score", [(0, 2.0), (1, 3.0)]) def test_likert_scoring_offsets_the_level_by_the_scale_minimum(self, minimum, score): strategy = strategy_with(level_payload(2), LikertScoring(5, minimum=minimum)) @@ -623,6 +775,46 @@ def test_invalid_answer_reports_the_evaluator_error_target(self): asyncio.run(strategy_with(payload, LikertScoring(5)).evaluate({})) assert raised.value.target == ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR + @pytest.mark.parametrize("levels", [[0] * 5, [0.1] * 5, [0.3] * 5]) + @pytest.mark.parametrize("minimum", [0, 1]) + def test_invalid_distribution_is_system_error_before_argmax(self, levels, minimum): + payload = decision_payload(0.95, {"tool_call_accuracy": score_answer(levels)}) + with pytest.raises(EvaluationException, match="sum to one") as raised: + asyncio.run(strategy_with(payload, LikertScoring(5, minimum=minimum)).evaluate({})) + assert raised.value.blame == ErrorBlame.SYSTEM_ERROR + assert raised.value.category == ErrorCategory.FAILED_EXECUTION + assert raised.value.target == ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR + assert raised.value.properties["total_tokens"] == 123 + assert raised.value.properties["model"] == "microsoft-decision-1" + + def test_public_call_rejects_invalid_distribution_with_evaluation_system_error(self, client): + class ScoredEvaluator(ModelEvaluatorBase): + _supports_decision_models = True + + def _derive_singleton_inputs(self): + return [["query", "response"]] + + async def _do_eval(self, eval_input): + return await DecisionEvaluationStrategy( + self._decision_model_runner, + result_key="tool_call_accuracy", + error_target=ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR, + questions=QUESTIONS, + scoring=LikertScoring(5), + threshold=3, + timeout=self._LLM_CALL_TIMEOUT, + ).evaluate(eval_input) + + payload = decision_payload(0.95, {"tool_call_accuracy": score_answer([0.3] * 5)}) + client.response = FakeResponse(body=json.dumps(payload).encode()) + evaluator = ScoredEvaluator(result_key="tool_call_accuracy", model_config=DECISION_CONFIG) + with pytest.raises(EvaluationException, match="sum to one") as raised: + evaluator(query="synthetic", response="synthetic") + assert raised.value.blame == ErrorBlame.SYSTEM_ERROR + assert raised.value.category == ErrorCategory.FAILED_EXECUTION + assert raised.value.target == ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR + assert raised.value.properties["total_tokens"] == 123 + def test_extra_properties_are_added_with_the_decision_provenance(self): strategy = strategy_with( level_payload(4), LikertScoring(5), extra_properties=lambda eval_input: {"calls": len(eval_input["calls"])} @@ -698,7 +890,7 @@ def test_applicability_cutoff_remains_fixed(self, status, expected): payload["answers"]["status"] ) - def test_successful_properties_preserve_common_metadata_shape(self): + def test_successful_properties_only_contain_decision_metadata(self): properties = asyncio.run(strategy_with(level_payload(4), LikertScoring(5)).evaluate({}))[ "tool_call_accuracy_properties" ] @@ -707,21 +899,119 @@ def test_successful_properties_preserve_common_metadata_shape(self): "prompt_tokens", "completion_tokens", "total_tokens", - "finish_reason", "model", - "sample_input", - "sample_output", } - assert {key: value for key, value in properties.items() if key != "decision_model"} == ( - ModelEvaluatorBase._get_token_metadata( - { - "input_token_count": 120, - "output_token_count": 3, - "total_token_count": 123, - "model_id": "microsoft-decision-1", - } - ) + assert {key: value for key, value in properties.items() if key != "decision_model"} == { + "prompt_tokens": 120, + "completion_tokens": 3, + "total_tokens": 123, + "model": "microsoft-decision-1", + } + + def test_likert_provenance_omits_legend_without_mutating_shared_response(self): + payload = level_payload(4) + answer = payload["answers"]["tool_call_accuracy"] + answer["legend"] = {str(index): "Repeated rubric description " * 20 for index in range(5)} + strategy = strategy_with(payload, LikertScoring(5)) + result = asyncio.run(strategy.evaluate({})) + native = result["tool_call_accuracy_properties"]["decision_model"]["native_answer"] + assert native == {name: value for name, value in answer.items() if name != "legend"} + assert "legend" in answer + assert result["tool_call_accuracy"] == 5 + assert result["tool_call_accuracy_properties"]["total_tokens"] == 123 + + +class TestDecisionConversationUsage: + def test_public_split_conversation_sums_native_usage_and_retains_model(self, client): + import logging + + from azure.ai.evaluation._evaluate._evaluate import _convert_single_row_to_aoai_format + + class SplitEvaluator(ModelEvaluatorBase): + _supports_decision_models = True + + def _derive_singleton_inputs(self): + return [["query", "response"]] + + async def _do_eval(self, eval_input): + return await DecisionEvaluationStrategy( + self._decision_model_runner, + result_key="tool_call_accuracy", + error_target=ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR, + questions=QUESTIONS, + scoring=LikertScoring(5), + threshold=3, + timeout=self._LLM_CALL_TIMEOUT, + ).evaluate(eval_input) + + payloads = [level_payload(0), level_payload(4)] + payloads[0]["usage"] = {"input_tokens": 12, "output_tokens": 3} + payloads[1]["usage"] = {"input_tokens": 34, "output_tokens": 5} + client.post = AsyncMock(side_effect=[FakeResponse(body=json.dumps(payload).encode()) for payload in payloads]) + evaluator = SplitEvaluator(result_key="tool_call_accuracy", model_config=DECISION_CONFIG) + result = evaluator( + conversation={ + "messages": [ + {"role": "user", "content": "first"}, + {"role": "assistant", "content": "one"}, + {"role": "user", "content": "second"}, + {"role": "assistant", "content": "two"}, + ] + } + ) + assert client.post.await_count == 2 + assert result["tool_call_accuracy"] == 3 + assert result["tool_call_accuracy_prompt_tokens"] == 46 + assert result["tool_call_accuracy_completion_tokens"] == 8 + assert result["tool_call_accuracy_total_tokens"] == 54 + assert result["tool_call_accuracy_model"] == "microsoft-decision-1" + assert result["evaluation_per_turn"]["tool_call_accuracy_prompt_tokens"] == [12, 34] + assert result["evaluation_per_turn"]["tool_call_accuracy_model"] == ["microsoft-decision-1"] * 2 + assert [json.loads(call.kwargs["content"])["state"]["query"] for call in client.post.await_args_list] == [ + "first", + "second", + ] + converted = _convert_single_row_to_aoai_format( + {f"outputs.judge.{key}": value for key, value in result.items()}, + 0, + None, + None, + 0, + {"judge": {"metrics": ["tool_call_accuracy"], "type": "azure_ai_evaluator"}}, + None, + logging.getLogger("test"), + ) + persisted = converted["results"][0] + assert persisted["sample"]["usage"] == {"prompt_tokens": 46, "completion_tokens": 8, "total_tokens": 54} + assert persisted["sample"]["model"] == "microsoft-decision-1" + + @pytest.mark.parametrize("models", [("first", "second"), ("first", None)]) + def test_different_or_unknown_models_remain_per_turn(self, models): + evaluator = ProbeModelEvaluator(result_key="probe", model_config=DECISION_CONFIG) + turns = [{"probe": 1, "probe_model": model} for model in models] + result = evaluator._aggregate_results(turns) + assert "probe_model" not in result + assert result["evaluation_per_turn"]["probe_model"] == list(models) + assert "probe_prompt_tokens" not in result + + def test_only_known_usage_is_summed_without_fabricating_totals(self): + evaluator = ProbeModelEvaluator(result_key="probe", model_config=DECISION_CONFIG) + result = evaluator._aggregate_results( + [{"probe": 1, "probe_prompt_tokens": 0}, {"probe": 5, "probe_completion_tokens": 4}] ) + assert result["probe"] == 3 + assert result["probe_prompt_tokens"] == 0 + assert result["probe_completion_tokens"] == 4 + assert "probe_total_tokens" not in result + + def test_chat_aggregation_is_unchanged(self): + with patch.object(_model_runners.AsyncPrompty, "load"): + evaluator = ProbeModelEvaluator(result_key="probe", prompty_file="probe.prompty", model_config=CHAT_CONFIG) + turns = [ + {"probe": 1, "probe_prompt_tokens": 12, "probe_model": "chat"}, + {"probe": 5, "probe_prompt_tokens": 34, "probe_model": "chat"}, + ] + assert evaluator._aggregate_results(turns) == EvaluatorBase._aggregate_results(evaluator, turns) class TestDecisionExecutionControls: @@ -930,7 +1220,7 @@ async def post(self, *args, **kwargs): finally: cleaned_up.append(True) - monkeypatch.setattr(_model_runners, "get_async_http_client", WaitingClient) + monkeypatch.setattr(_model_runners, "get_async_http_client", lambda **_: WaitingClient()) async def request(): cancelled = False @@ -1013,7 +1303,7 @@ async def request(): assert cleaned_up == [True] and not client.calls @pytest.mark.parametrize("termination", ["timeout", "cancelled"]) - def test_queued_default_auth_does_not_create_an_unawaited_coroutine(self, client, termination): + def test_queued_explicit_auth_is_not_started_after_timeout_or_cancellation(self, client, termination): config = {key: value for key, value in DECISION_CONFIG.items() if key != "api_key"} entered = threading.Event() release = threading.Event() @@ -1032,15 +1322,16 @@ async def request(): while not entered.is_set(): await asyncio.sleep(0) handle = loop.call_later(0.01, cancel.set) - with patch.object(DecisionModelRunner, "_get_default_token", new_callable=AsyncMock) as get_token: - with pytest.raises(EvaluationException) as raised: - await DecisionModelRunner( - config, cancellation_requested=cancel.is_set if termination == "cancelled" else None - )(state={}, questions=QUESTIONS, timeout=0.03 if termination == "timeout" else 1) - assert raised.value.blame == ( - ErrorBlame.SYSTEM_ERROR if termination == "timeout" else ErrorBlame.USER_ERROR - ) - get_token.assert_not_called() + credential = Mock(get_token=Mock(return_value=SimpleNamespace(token="unused"))) + with pytest.raises(EvaluationException) as raised: + await DecisionModelRunner( + config, credential, cancellation_requested=cancel.is_set if termination == "cancelled" else None + )(state={}, questions=QUESTIONS, timeout=0.03 if termination == "timeout" else 1) + assert raised.value.blame == ( + ErrorBlame.SYSTEM_ERROR if termination == "timeout" else ErrorBlame.USER_ERROR + ) + credential.get_token.assert_not_called() + credential.close.assert_not_called() finally: if handle is not None: handle.cancel() @@ -1090,7 +1381,7 @@ async def send(self, request, **kwargs): cleaned_up.append(True) transport = Transport() - monkeypatch.setattr(_model_runners, "get_async_http_client", lambda: AsyncHttpPipeline(transport=transport)) + monkeypatch.setattr(_model_runners, "get_async_http_client", lambda **_: AsyncHttpPipeline(transport=transport)) async def request(): nonlocal cancelled @@ -1140,7 +1431,7 @@ async def send(self, request, **kwargs): raise ServiceRequestError("transport-secret") transport = Transport() - monkeypatch.setattr(_model_runners, "get_async_http_client", lambda: AsyncHttpPipeline(transport=transport)) + monkeypatch.setattr(_model_runners, "get_async_http_client", lambda **_: AsyncHttpPipeline(transport=transport)) with pytest.raises(EvaluationException) as raised: run(DecisionModelRunner(DECISION_CONFIG, max_retries=max_retries)) expected = max_retries + 1 @@ -1177,6 +1468,6 @@ async def send(self, request, **kwargs): return Response(request) transport = Transport() - monkeypatch.setattr(_model_runners, "get_async_http_client", lambda: AsyncHttpPipeline(transport=transport)) + monkeypatch.setattr(_model_runners, "get_async_http_client", lambda **_: AsyncHttpPipeline(transport=transport)) assert run(DecisionModelRunner(DECISION_CONFIG))["usage"] == {"input_tokens": 120, "output_tokens": 3} assert transport.calls == 1 From 061f8d9fa6cf374daa5adbf857f8b4f56eca3334 Mon Sep 17 00:00:00 2001 From: salma-elshafey Date: Fri, 16 Oct 2026 19:43:23 +0300 Subject: [PATCH 18/24] Describe unreleased Decision support as feature capabilities Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e564c5c8-480e-48e7-a9e0-44450f774b5e --- sdk/evaluation/azure-ai-evaluation/CHANGELOG.md | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index c7c673bb6580..ed63eeb1db95 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -4,23 +4,20 @@ ### Features Added -- Added shared model runners and evaluation strategies for asset evaluators using decision models in Microsoft Foundry. -- Decision evaluator results expose known final-response token usage and model fields in standard evaluation rows, including skipped evaluations, while retaining known metadata on failures. Usage does not include earlier retry attempts. +- Added shared model runners and evaluation strategies for asset evaluators using Decision models in Microsoft Foundry, including Output Quality and Tool Use Quality aggregate and member metrics. +- Decision authentication uses an explicit `api_key` or caller-owned credential with a callable `get_token`. Reuse a synchronous `azure.identity.DefaultAzureCredential` for synchronous or batch calls and close it afterward; async credentials require the same caller-managed event loop. API keys take precedence; Prompty authentication is unchanged. +- Decision score probabilities must sum to one within an absolute `1e-6` tolerance; invalid distributions raise a system-classified `EvaluationException`. +- Decision results expose known final-response usage, model, and provenance on completed, skipped, and failed evaluations, counting shared requests once and summing split-turn usage. Usage excludes earlier retry attempts. +- Decision native async HTTP supports overall deadlines and cancellation during response body reads. TLS uses Python/OpenSSL trust settings such as `SSL_CERT_FILE`, not `REQUESTS_CA_BUNDLE`; shared Prompty transport behavior is unchanged. ### Breaking Changes -- Decision evaluators now require an explicit `api_key` or a credential with a callable `get_token`; implicit default authentication is no longer selected. For Entra authentication, create and reuse a synchronous `azure.identity.DefaultAzureCredential`, pass it as `credential=`, and close it after all synchronous or batch evaluations finish. Async credentials must remain on the same caller-managed event loop. API-key precedence and Prompty authentication are unchanged. - ### Bugs Fixed -- Decision score answers now reject probability distributions whose sum differs from one by more than `1e-6`, reporting a system error instead of selecting a score from an invalid distribution. - Hardened local Prompty image resolution so only relative files within the Prompty directory are inlined, with opened-file validation to prevent path changes from redirecting reads and without exposing the local directory in fallback logs. -- Output Quality and Tool Use Quality asset evaluators retain their aggregate and member metrics when used with `evaluate` without catalog metadata. ### Other Changes -- Decision requests use native async HTTP so overall timeout and cancellation stop response body reads. This path uses Python/OpenSSL TLS trust settings (for example, `SSL_CERT_FILE`), not the Requests-specific `REQUESTS_CA_BUNDLE`. Shared Prompty transport behavior is unchanged. - ## 1.18.9 (2026-10-12) ### Bugs Fixed From a69217c33402ce432dafbb2cf7bc43e1246240c5 Mon Sep 17 00:00:00 2001 From: salma-elshafey Date: Fri, 16 Oct 2026 20:13:03 +0300 Subject: [PATCH 19/24] Include verified Decision deployment error code Expose only the verified DeploymentNotFound code while retaining generic errors for all other provider content. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e564c5c8-480e-48e7-a9e0-44450f774b5e --- .../_evaluators/_common/_model_runners.py | 6 +- .../unittests/test_decision_model_support.py | 66 +++++++++++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py index d0b97a9b0f2d..67d59f4f801d 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py @@ -44,6 +44,7 @@ def value(self) -> str: _DECISION_PATH = "/providers/microsoft/v1/systemone" _DECISION_TOKEN_SCOPE = "https://cognitiveservices.azure.com/.default" _DECISION_RETRY_STATUS_CODES = [408, 429, 500, 502, 503, 504, 529] +_DECISION_SAFE_ERROR_CODES = {"DeploymentNotFound"} _DECISION_CONFIG_KEYS = { "type", "azure_endpoint", @@ -471,7 +472,10 @@ async def _post(self, content: bytes, *, deadline: float) -> Any: payload = None if status != 200: blame = ErrorBlame.USER_ERROR if status in (400, 401, 403, 404, 422) else ErrorBlame.SYSTEM_ERROR - error = _failed(f"The decision model returned HTTP {status}.", blame) + provider_error = payload.get("error") if isinstance(payload, dict) else None + code = provider_error.get("code") if isinstance(provider_error, dict) else None + diagnostic = f" ({code})" if isinstance(code, str) and code in _DECISION_SAFE_ERROR_CODES else "" + error = _failed(f"The decision model returned HTTP {status}{diagnostic}.", blame) error.properties = self._response_properties(payload) raise error return payload diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py index 879c6f62b301..ec8e64b25667 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py @@ -400,6 +400,72 @@ def test_http_failures_are_sanitized(self, client, status, blame): assert str(status) in str(raised.value) assert "test-key-value" not in str(raised.value) and "secret detail" not in str(raised.value) + @pytest.mark.parametrize("status", [404, 500]) + @pytest.mark.parametrize("response_limit", [1, 1024]) + def test_verified_http_error_code_preserves_classification_and_accepted_metadata( + self, client, status, response_limit + ): + payload = decision_payload() + payload["error"] = { + "code": "DeploymentNotFound", + "message": "test-key-value and private-message", + "details": "private-details", + } + client.response = FakeResponse(status, json.dumps(payload).encode()) + with pytest.raises(EvaluationException) as raised: + run(DecisionModelRunner(DECISION_CONFIG, max_retries=0, max_response_bytes=response_limit)) + diagnostic = " (DeploymentNotFound)" if response_limit == 1024 else "" + prefix = "UserError" if status == 404 else "InternalError" + assert str(raised.value) == f"({prefix}) The decision model returned HTTP {status}{diagnostic}." + assert raised.value.blame == (ErrorBlame.USER_ERROR if status == 404 else ErrorBlame.SYSTEM_ERROR) + assert raised.value.category == ErrorCategory.FAILED_EXECUTION + assert raised.value.target == ErrorTarget.EVALUATE + assert raised.value.properties == ( + { + "decision_model": {"deployment": "Decision-1"}, + "prompt_tokens": 120, + "completion_tokens": 3, + "total_tokens": 123, + "model": "microsoft-decision-1", + } + if response_limit == 1024 + else {"decision_model": {"deployment": "Decision-1"}} + ) + + @pytest.mark.parametrize( + "payload", + [ + {"error": {"code": "secret_like_identifier_123"}}, + {"error": {"code": "UnknownProviderCode"}}, + {"error": {"code": "deploymentnotfound"}}, + {"error": {"code": "DeploymentNotFound "}}, + {"error": {"code": None}}, + {"error": {"code": 404}}, + {"error": {"code": True}}, + {"error": {"code": ["DeploymentNotFound"]}}, + {"error": {"code": {"value": "DeploymentNotFound"}}}, + {"error": {"message": "private-message", "details": "private-details"}}, + {"error": "DeploymentNotFound"}, + {"error": ["DeploymentNotFound"]}, + {"error": None}, + {"code": "DeploymentNotFound"}, + ["DeploymentNotFound"], + None, + ], + ) + def test_unverified_or_malformed_http_error_codes_are_not_exposed(self, client, payload): + client.response = FakeResponse(404, json.dumps(payload).encode()) + with pytest.raises(EvaluationException) as raised: + run(DecisionModelRunner(DECISION_CONFIG)) + assert str(raised.value) == "(UserError) The decision model returned HTTP 404." + assert raised.value.blame == ErrorBlame.USER_ERROR + + def test_non_json_http_error_cannot_surface_verified_code_text(self, client): + client.response = FakeResponse(404, b"DeploymentNotFound private-message test-key-value") + with pytest.raises(EvaluationException) as raised: + run(DecisionModelRunner(DECISION_CONFIG)) + assert str(raised.value) == "(UserError) The decision model returned HTTP 404." + def test_transport_failure_is_sanitized(self, monkeypatch): class Broken(FakeClient): async def post(self, *_, **__): From a0af183f8173b3794bec7e19f2c60934917a8467 Mon Sep 17 00:00:00 2001 From: salma-elshafey Date: Sat, 17 Oct 2026 23:39:27 +0300 Subject: [PATCH 20/24] Add Decision execution telemetry and preserve failure diagnostics Bind context-aware execution logging and optional metrics, preserve validated shared and independent execution metadata, and accept precision-aware probability rounding. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e564c5c8-480e-48e7-a9e0-44450f774b5e --- .../azure-ai-evaluation/CHANGELOG.md | 3 +- .../_batch_run/_run_submitter_client.py | 16 +- .../_common/_decision_execution.py | 228 +++++++++++++++ .../_evaluators/_common/_decision_strategy.py | 4 + .../_evaluators/_common/_model_runners.py | 64 ++++- .../test_decision_batch_persistence.py | 95 +++++++ .../test_decision_default_transport.py | 23 +- .../unittests/test_decision_model_support.py | 268 +++++++++++++++++- 8 files changed, 677 insertions(+), 24 deletions(-) create mode 100644 sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_execution.py diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index ed63eeb1db95..ef1795755b10 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -6,8 +6,9 @@ - Added shared model runners and evaluation strategies for asset evaluators using Decision models in Microsoft Foundry, including Output Quality and Tool Use Quality aggregate and member metrics. - Decision authentication uses an explicit `api_key` or caller-owned credential with a callable `get_token`. Reuse a synchronous `azure.identity.DefaultAzureCredential` for synchronous or batch calls and close it afterward; async credentials require the same caller-managed event loop. API keys take precedence; Prompty authentication is unchanged. -- Decision score probabilities must sum to one within an absolute `1e-6` tolerance; invalid distributions raise a system-classified `EvaluationException`. +- Decision score probabilities use precision-aware sum validation: hundredth-precision values allow `0.005` rounding error per level plus `1e-12`; other values use an absolute `0.001` tolerance. Invalid distributions raise a system-classified `EvaluationException`; probabilities are not normalized. - Decision results expose known final-response usage, model, and provenance on completed, skipped, and failed evaluations, counting shared requests once and summing split-turn usage. Usage excludes earlier retry attempts. +- Decision requests provide context-aware per-attempt and terminal execution observations with optional OpenTelemetry metrics. Validated shared and independent execution summaries and their distinct usage scopes are preserved in failed-row metadata. - Decision native async HTTP supports overall deadlines and cancellation during response body reads. TLS uses Python/OpenSSL trust settings such as `SSL_CERT_FILE`, not `REQUESTS_CA_BUNDLE`; shared Prompty transport behavior is unchanged. ### Breaking Changes diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py index 34cbd602475c..bd3226987c2a 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py @@ -23,6 +23,7 @@ from ..._evaluate._utils import _has_aggregator from ..._constants import Prefixes, PF_BATCH_TIMEOUT_SEC from ..._exceptions import EvaluationException +from ..._evaluators._common._decision_execution import validated_execution from .._utils import get_int_env_var as get_int @@ -31,7 +32,7 @@ def _decision_error_metadata(exception: EvaluationException) -> Dict[str, Any]: - """Keep only known final-response usage and validated provenance for failed Decision rows.""" + """Keep known usage and validated provenance, preserving the producer's distinct usage scopes.""" properties: Dict[str, Any] = {} semantic_properties = exception.properties or {} for key in ("prompt_tokens", "completion_tokens", "total_tokens"): @@ -57,6 +58,19 @@ def _decision_error_metadata(exception: EvaluationException) -> Dict[str, Any]: and 0 <= value <= 1 ): provenance[key] = value + for key, accepted in ( + ("backend", ("foundry_decision", "system_one")), + ("protocol", ("shared_sdk", "system_one")), + ("result_usage_scope", ("reported_attempts_only", "final_response_only")), + ): + value = native.get(key) + if isinstance(value, str) and value in accepted: + provenance[key] = value + if isinstance(native.get("usage_available"), bool): + provenance["usage_available"] = native["usage_available"] + execution = validated_execution(native.get("execution")) + if execution is not None: + provenance["execution"] = execution properties["decision_model"] = provenance return {"code": exception.category.value, "blame": exception.blame.value, "properties": properties} diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_execution.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_execution.py new file mode 100644 index 000000000000..a75005eb73c2 --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_execution.py @@ -0,0 +1,228 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- + +"""Bounded Decision execution metadata and request-owned observations.""" + +import json +import logging +import math +import re +import time +from typing import Any, Dict, Optional + +from azure.core.pipeline.policies import CustomHookPolicy + +LOGGER = logging.getLogger(__name__) +_PREFIX = "azure.ai.evaluation.decision." + + +def _integer(value: Any, maximum: Optional[int] = None) -> bool: + return ( + isinstance(value, int) and not isinstance(value, bool) and 0 <= value and (maximum is None or value <= maximum) + ) + + +def validated_execution(value: Any) -> Optional[Dict[str, Any]]: + """Copy only a complete, strictly typed shared or independent execution summary. + + :param value: The untrusted summary carrier. + :type value: Any + :return: Known validated fields, or None for an invalid carrier. + :rtype: Optional[Dict[str, Any]] + """ + if not isinstance(value, dict): + return None + if "operation_id" in value: + keys = { + "operation_id", + "attempt_count", + "unknown_usage_attempt_count", + "result_usage_scope", + "usage_completeness", + } + if set(value) != keys or not all( + ( + isinstance(value["operation_id"], str) + and re.fullmatch("[0-9a-f]{32}", value["operation_id"]) is not None, + _integer(value["attempt_count"]), + _integer(value["unknown_usage_attempt_count"]), + value["result_usage_scope"] == "final_response_only", + value["usage_completeness"] + in ("all_attempts_reported", "some_attempts_reported", "no_attempts_reported"), + ) + ): + return None + return dict(value) + keys = {"attempt_count", "retry_count", "elapsed_ms", "outcome", "last_http_status", "usage_state"} + elapsed = value.get("elapsed_ms") + status = value.get("last_http_status") + if ( + not keys - {"last_http_status"} <= value.keys() + or isinstance(elapsed, bool) + or not isinstance(elapsed, (int, float)) + ): + return None + if not all( + ( + _integer(value["attempt_count"], 21), + _integer(value["retry_count"], 20), + 0 <= elapsed <= 86400000 and math.isfinite(elapsed), + value["outcome"] in ("completed", "error", "deadline", "cancelled"), + value["usage_state"] in ("known", "partial", "unknown"), + status is None or (_integer(status, 599) and status >= 100), + ) + ): + return None + return {key: value.get(key) for key in keys} + + +def _diagnostic(message: str) -> None: + # Observer failures must not replace a model result, even if diagnostic logging is also broken. + try: + LOGGER.warning(message) + except Exception: # pylint: disable=broad-except + pass + + +class DecisionExecution(CustomHookPolicy): + """One operation's per-attempt pipeline policy; never shared between concurrent calls.""" + + def __init__(self, logger: Any, max_response_bytes: Optional[int]) -> None: + super().__init__() + self.logger = logger + self.max_response_bytes = max_response_bytes + self.started = time.monotonic() + self.attempt_started: Optional[float] = None + self.attempt_count = 0 + self.known_usage_attempts = 0 + self.partial_usage_attempts = 0 + self.last_http_status: Optional[int] = None + self.outcome = "error" + self.instruments: Optional[Dict[str, Any]] = None + + def on_request(self, request: Any) -> None: + self.attempt_count += 1 + self.attempt_started = time.monotonic() + super().on_request(request) + + def on_response(self, request: Any, response: Any) -> None: + raw = response.http_response + status = raw.status_code + self.last_http_status = status if _integer(status, 599) and status >= 100 else None + known = False + partial = False + try: + body = raw.body() + if self.max_response_bytes is None or len(body) <= self.max_response_bytes: + payload = json.loads(body) + usage = payload.get("usage") if isinstance(payload, dict) else None + if isinstance(usage, dict): + tokens = [_integer(usage.get(key)) for key in ("input_tokens", "output_tokens")] + known, partial = all(tokens), any(tokens) + except (ValueError, TypeError): + pass + self.known_usage_attempts += int(known) + self.partial_usage_attempts += int(partial and not known) + self.finish_attempt( + "completed" if status == 200 else "error", + "known" if known else "partial" if partial else "unknown", + self.last_http_status, + ) + super().on_response(request, response) + + def on_exception(self, request: Any) -> None: + self.finish_attempt("error") + + def summary(self, outcome: str) -> Dict[str, Any]: + return { + "attempt_count": self.attempt_count, + "retry_count": max(0, self.attempt_count - 1), + "elapsed_ms": (time.monotonic() - self.started) * 1000, + "outcome": outcome, + "last_http_status": self.last_http_status, + "usage_state": ( + "known" + if self.attempt_count and self.known_usage_attempts == self.attempt_count + else "partial" if self.known_usage_attempts or self.partial_usage_attempts else "unknown" + ), + } + + def finish_attempt(self, outcome: str, usage: str = "unknown", status: Optional[int] = None) -> None: + if self.attempt_started is None: + return + elapsed = (time.monotonic() - self.attempt_started) * 1000 + self.attempt_started = None + self.emit("attempt", outcome, self.summary(outcome), elapsed, usage, status) + + def finish(self, outcome: str) -> Optional[Dict[str, Any]]: + self.finish_attempt(outcome) + summary = self.summary(outcome) + self.emit("execution", outcome, summary, summary["elapsed_ms"], summary["usage_state"]) + accepted = validated_execution(summary) + if accepted is None: + _diagnostic("Decision execution summary exceeds the hosted metadata bounds; summary omitted.") + return accepted + + def emit( + self, + phase: str, + outcome: str, + summary: Dict[str, Any], + elapsed: float, + usage: str, + status: Optional[int] = None, + ) -> None: + status = self.last_http_status if phase == "execution" else status + dimensions = { + "backend": "decision", + "route": "shared_sdk", + "phase": phase, + "outcome": outcome, + "usage_state": usage, + "http_status_class": f"{status // 100}xx" if status is not None else "unknown", + "throttled": str(status == 429).lower(), + } + try: + logger = self.logger + if logger is not None and logger.isEnabledFor(logging.INFO): + fields = { + "decision_dimensions": json.dumps(dimensions), + "decision_execution": json.dumps(summary), + } + while isinstance(logger, logging.LoggerAdapter): + # Python 3.10 LoggerAdapter.process replaces, rather than merges, per-call extra fields. + _, kwargs = logger.process("DECISION_EXECUTION_OBSERVATION", {"extra": dict(fields)}) + fields = {**kwargs.get("extra", {}), **fields} + logger = logger.logger + logger.info("DECISION_EXECUTION_OBSERVATION", extra=fields) + except Exception: # pylint: disable=broad-except + _diagnostic("Decision execution log sink failed; evaluation and metrics continue.") + try: + self.emit_metrics(phase, elapsed, dimensions) + except Exception: # pylint: disable=broad-except + _diagnostic("Decision execution metric sink failed; evaluation and logs continue.") + + def emit_metrics(self, phase: str, elapsed: float, dimensions: Dict[str, str]) -> None: + if self.instruments is None: + try: + from opentelemetry.metrics import get_meter # pylint: disable=import-outside-toplevel + except ImportError: + self.instruments = {} + LOGGER.debug("Decision metrics require the optional opentelemetry extra.") + return + meter = get_meter("azure.ai.evaluation.decision") + self.instruments = { + key: meter.create_counter(_PREFIX + key) + for key in ("provider.attempts", "provider.retries", "operations") + } + self.instruments["provider.duration"] = meter.create_histogram(_PREFIX + "provider.duration", unit="ms") + if not self.instruments: + return + if phase == "attempt": + self.instruments["provider.attempts"].add(1, attributes=dimensions) + self.instruments["provider.duration"].record(elapsed, attributes=dimensions) + if self.attempt_count > 1: + self.instruments["provider.retries"].add(1, attributes=dimensions) + else: + self.instruments["operations"].add(1, attributes=dimensions) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py index 6c31c72063c7..457a2f0723b8 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py @@ -16,6 +16,7 @@ from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException from ._base_model_eval import not_applicable_result from ._model_runners import DecisionModelRunner, validate_noul_answer, validate_score_answer +from ._decision_execution import validated_execution STATUS_CUTOFF = 0.5 T = TypeVar("T") @@ -194,6 +195,9 @@ def parse(self, response: Dict[str, Any], eval_input: Dict[str, Any]) -> Dict[st "status_probability_threshold": STATUS_CUTOFF, "reason_type": "templated", } + execution = validated_execution(response.get("_decision_execution")) + if execution is not None: + properties["decision_model"]["execution"] = execution tokens = decision_token_counts(response) properties.update( prompt_tokens=tokens["input_token_count"], diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py index 67d59f4f801d..8b5050fc76b6 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py @@ -12,6 +12,7 @@ import inspect import ipaddress import json +import logging import math import os import time @@ -24,6 +25,7 @@ from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException from ..._common.utils import construct_prompty_model_config, validate_model_config from ..._http_utils import get_async_http_client +from ._decision_execution import DecisionExecution if os.getenv("AI_EVALS_USE_PF_PROMPTY", "false").lower() == "true": from promptflow.core._flow import AsyncPrompty @@ -54,7 +56,6 @@ def value(self) -> str: "probability_threshold", } _DEFAULT_PROBABILITY_THRESHOLD = 0.75 -_PROBABILITY_SUM_ABS_TOLERANCE = 1e-6 # The service adds api_version to every resolved model configuration; the decision endpoint does not use it. _IGNORED_CONFIG_KEYS = {"api_version"} @@ -141,7 +142,8 @@ def validate_noul_answer(answer: Dict[str, Any]) -> float: def validate_score_answer(answer: Dict[str, Any], levels: int) -> List[float]: """Return the level probabilities of a ``score`` answer after validating its fields. - Probabilities must sum to one within an absolute tolerance of ``1e-6`` for decimal rounding. + Hundredth-precision probabilities allow rounding error of ``0.005`` per level plus ``1e-12``. + Other distributions must sum to one within an absolute tolerance of ``0.001``. The native mean is checked for finiteness, not consistency with the distribution. :param answer: The answer for a ``score`` question. @@ -156,8 +158,9 @@ def validate_score_answer(answer: Dict[str, Any], levels: int) -> List[float]: if not isinstance(probabilities, dict) or set(probabilities) != {str(i) for i in range(levels)}: raise ValueError(f"a score answer requires probabilities for exactly {levels} levels.") distribution = [_probability(probabilities[str(i)]) for i in range(levels)] - if not math.isclose(math.fsum(distribution), 1.0, rel_tol=0.0, abs_tol=_PROBABILITY_SUM_ABS_TOLERANCE): - raise ValueError("score probabilities must sum to one within an absolute tolerance of 1e-6.") + tolerance = len(distribution) * 0.005 + 1e-12 if all(value == round(value, 2) for value in distribution) else 0.001 + if not math.isclose(math.fsum(distribution), 1.0, rel_tol=0.0, abs_tol=tolerance): + raise ValueError("score probabilities must sum to one within the precision-aware rounding tolerance.") mean = answer.get("score") if isinstance(mean, bool) or not isinstance(mean, (int, float)) or not math.isfinite(mean): raise ValueError("the score mean must be finite.") @@ -287,7 +290,7 @@ def __init__( credential = credential if credential is not None else model_config.get("credential") if not api_key and not callable(getattr(credential, "get_token", None)): raise _invalid_config("Decision models require an 'api_key' or a credential with a callable 'get_token'.") - self._credential = None if api_key else credential + self._credential: Any = None if api_key else credential threshold = model_config.get("probability_threshold", _DEFAULT_PROBABILITY_THRESHOLD) if "probability_threshold" in model_config and not binary: @@ -301,6 +304,15 @@ def __init__( self._deployment = deployment self._api_key = api_key self._user_agent = f"{UserAgentSingleton().value} (type=evaluator subtype={evaluator_name})" + self._execution_logger: Any = logging.getLogger(__name__) + + def _bind_execution_logger(self, logger: Any) -> None: + """Bind a host's context-aware logger without changing transport or credential ownership. + + :param logger: The host logger or LoggerAdapter; None disables execution logs, not metrics. + :type logger: Any + """ + self._execution_logger = logger @property def deployment(self) -> str: @@ -327,11 +339,13 @@ async def __call__(self, *, state: Any, questions: Dict[str, Dict[str, Any]], ti ): raise _invalid_config("'timeout' must be a positive finite number.") deadline = time.monotonic() + timeout - for question in questions.values(): - if question.get("type") not in ("noul", "score"): - raise _invalid_config(f"Decision question type '{question.get('type')}' is not supported.") + execution = DecisionExecution(self._execution_logger, self._max_response_bytes) + outcome = "error" properties = self._response_properties(None) try: + for question in questions.values(): + if question.get("type") not in ("noul", "score"): + raise _invalid_config(f"Decision question type '{question.get('type')}' is not supported.") body = {"model": self._deployment, "state": state, "questions": questions} try: content = json.dumps(body, ensure_ascii=False, separators=(",", ":"), allow_nan=False).encode("utf-8") @@ -361,19 +375,36 @@ async def __call__(self, *, state: Any, questions: Dict[str, Dict[str, Any]], ti raise _failed("The decision request exceeds the configured byte budget.", ErrorBlame.USER_ERROR) try: payload = await asyncio.wait_for( - self._await_request(content, deadline), timeout=max(0, deadline - time.monotonic()) + self._await_request(content, deadline, execution), timeout=max(0, deadline - time.monotonic()) ) except asyncio.TimeoutError: + outcome = "deadline" raise _failed("The decision model request timed out.") from None properties = self._response_properties(payload) try: - return self._validate_response(payload, questions) + result = self._validate_response(payload, questions) except ValueError as exc: raise _failed(f"The decision model returned an invalid response: {exc}") from None + outcome = "completed" + return result + except asyncio.CancelledError as exc: + outcome = "cancelled" + setattr(exc, "properties", properties) + raise except EvaluationException as exc: + if outcome != "deadline": + outcome = execution.outcome if exc.properties is None: exc.properties = properties + properties = exc.properties raise + finally: + summary = execution.finish(outcome) + if summary is not None: + if outcome == "completed": + result["_decision_execution"] = summary + else: + properties.setdefault("decision_model", {})["execution"] = summary def _cancelled(self) -> bool: callback = self._cancellation_requested @@ -387,15 +418,17 @@ def _cancelled(self) -> bool: raise _failed("The decision cancellation callback must return a boolean.", ErrorBlame.USER_ERROR) return result - async def _await_request(self, content: bytes, deadline: float) -> Any: + async def _await_request(self, content: bytes, deadline: float, execution: DecisionExecution) -> Any: if self._cancelled(): + execution.outcome = "cancelled" raise _failed("The decision model request was cancelled.", ErrorBlame.USER_ERROR) if self._cancellation_requested is None: - return await self._post(content, deadline=deadline) - task = asyncio.create_task(self._post(content, deadline=deadline)) + return await self._post(content, deadline=deadline, execution=execution) + task = asyncio.create_task(self._post(content, deadline=deadline, execution=execution)) try: while not task.done(): if self._cancelled(): + execution.outcome = "cancelled" raise _failed("The decision model request was cancelled.", ErrorBlame.USER_ERROR) await asyncio.wait({task}, timeout=0.05) return await task @@ -412,7 +445,7 @@ async def _get_token(self) -> str: token = await token return token.token - async def _post(self, content: bytes, *, deadline: float) -> Any: + async def _post(self, content: bytes, *, deadline: float, execution: DecisionExecution) -> Any: headers = {"x-ms-useragent": self._user_agent, "Content-Type": "application/json"} if self._api_key: headers["api-key"] = self._api_key @@ -425,6 +458,7 @@ async def _post(self, content: bytes, *, deadline: float) -> Any: raise _failed("Could not acquire a token for the decision model.", ErrorBlame.USER_ERROR) from None policies: Dict[str, Any] = { + "custom_hook_policy": execution, "retry_policy": AsyncRetryPolicy( retry_total=self._max_retries, retry_connect=self._max_retries, @@ -437,6 +471,7 @@ async def _post(self, content: bytes, *, deadline: float) -> Any: } remaining = deadline - time.monotonic() if remaining <= 0: + execution.outcome = "deadline" raise _failed("The decision model request timed out.") # Native retry policy carries this budget across attempts and clamps the scalar connect/read timeout. options: Dict[str, Any] = { @@ -457,6 +492,7 @@ async def _post(self, content: bytes, *, deadline: float) -> Any: raise except Exception: # pylint: disable=broad-except if time.monotonic() >= deadline: + execution.outcome = "deadline" raise _failed("The decision model request timed out.") from None raise _failed("The decision model request failed; check the endpoint and network access.") from None if self._max_response_bytes is not None and len(content) > self._max_response_bytes: diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py index f2dc3711757d..54276f05cc3a 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py @@ -14,6 +14,7 @@ from azure.ai.evaluation import F1ScoreEvaluator from azure.ai.evaluation._evaluate._batch_run import RunSubmitterClient +from azure.ai.evaluation._evaluate._batch_run._run_submitter_client import _decision_error_metadata from azure.ai.evaluation._evaluate._evaluate import ( _calculate_aoai_evaluation_summary, _convert_single_row_to_aoai_format, @@ -107,6 +108,92 @@ def strategy(runner): class TestDecisionBatchPersistence: + @pytest.mark.parametrize("backend", ["system_one", "foundry_decision"]) + @pytest.mark.parametrize("known", [False, True]) + @pytest.mark.parametrize("attempted", [False, True]) + def test_independent_failed_metadata_preserves_distinct_scopes_and_known_fields(self, backend, known, attempted): + native = { + "backend": backend, + "protocol": "system_one", + "usage_available": known, + "result_usage_scope": "reported_attempts_only", + "private": "never-persist", + } + execution = { + "operation_id": "abcdef0123456789" * 2, + "attempt_count": 1, + "unknown_usage_attempt_count": int(not known), + "result_usage_scope": "final_response_only", + "usage_completeness": "all_attempts_reported" if known else "no_attempts_reported", + } + if attempted: + native["execution"] = execution + properties = {"decision_model": native, "private": "never-persist"} + if known: + properties.update(prompt_tokens=41, completion_tokens=3, total_tokens=44, model="independent-model") + error = EvaluationException( + "synthetic failure", + category=ErrorCategory.FAILED_EXECUTION, + blame=ErrorBlame.SYSTEM_ERROR, + properties=properties, + ) + saved = _decision_error_metadata(error)["properties"] + provenance = saved["decision_model"] + assert provenance["backend"] == backend and provenance["protocol"] == "system_one" + assert provenance["usage_available"] is known + assert provenance["result_usage_scope"] == "reported_attempts_only" + assert "private" not in json.dumps(saved) + if attempted: + assert provenance["execution"] == execution + assert provenance["execution"]["result_usage_scope"] == "final_response_only" + else: + assert "execution" not in provenance + assert saved.get("total_tokens") == (44 if known else None) + + @pytest.mark.parametrize( + "changes", + [ + {"operation_id": "A" * 32}, + {"operation_id": "a" * 31}, + {"operation_id": 42}, + {"attempt_count": True}, + {"attempt_count": -1}, + {"unknown_usage_attempt_count": True}, + {"unknown_usage_attempt_count": -1}, + {"result_usage_scope": "reported_attempts_only"}, + {"usage_completeness": "unknown"}, + {"extra": "private"}, + ], + ) + def test_malformed_independent_summary_is_dropped_not_rewritten(self, changes): + execution = { + "operation_id": "a" * 32, + "attempt_count": 1, + "unknown_usage_attempt_count": 0, + "result_usage_scope": "final_response_only", + "usage_completeness": "all_attempts_reported", + } + error = EvaluationException( + "synthetic failure", + properties={"decision_model": {"execution": {**execution, **changes}, "usage_available": False}}, + ) + assert _decision_error_metadata(error)["properties"]["decision_model"] == {"usage_available": False} + + def test_failed_metadata_rejects_untyped_and_unrecognized_provenance(self): + error = EvaluationException( + "synthetic failure", + properties={ + "decision_model": { + "backend": "private", + "protocol": ["system_one"], + "usage_available": 1, + "result_usage_scope": True, + "execution": "private", + } + }, + ) + assert _decision_error_metadata(error)["properties"] == {"decision_model": {}} + @pytest.mark.parametrize("members", [5, 6]) @pytest.mark.parametrize("case", ["completed", "skipped", "mixed", "semantic_error"]) @pytest.mark.parametrize("primary_first", [False, True]) @@ -446,6 +533,14 @@ def cancel(): result = next(item for item in rows[0]["results"] if item["name"] == "judge") assert result["status"] == "error" and result["score"] is None and result["passed"] is None assert "decision_request_observations" not in result["properties"] + execution = result["properties"]["decision_model"].pop("execution") + assert execution["attempt_count"] == (0 if case == "precancel" else 1) + assert execution["outcome"] == ("cancelled" if "cancel" in case else "error") + assert execution["usage_state"] == ( + "known" + if case in ("malformed_shape", "billed_cancel") + else "partial" if case == "partial_usage" else "unknown" + ) if case == "malformed_shape": assert result["sample"]["usage"] == {"prompt_tokens": 12, "completion_tokens": 4, "total_tokens": 16} assert result["sample"]["model"] == "decision-native" diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_default_transport.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_default_transport.py index f1af01b2dfbb..d7fdb5fd56a5 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_default_transport.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_default_transport.py @@ -15,6 +15,7 @@ import requests from azure.ai.evaluation._evaluators._common._model_runners import DecisionModelRunner +from azure.ai.evaluation._evaluators._common._decision_execution import validated_execution from azure.ai.evaluation._evaluators._common._base_model_eval import ModelEvaluatorBase from azure.ai.evaluation._exceptions import ErrorBlame, EvaluationException from azure.ai.evaluation._http_utils import get_async_http_client @@ -73,6 +74,16 @@ def invoke(endpoint, timeout=3, **controls): return asyncio.run(runner(state={"query": "é"}, questions=QUESTIONS, timeout=timeout)) +def assert_failure_execution(error, attempts, outcome): + assert set(error.properties) == {"decision_model"} + native = error.properties["decision_model"] + assert set(native) == {"deployment", "execution"} and native["deployment"] == "Decision-1" + execution = native["execution"] + assert validated_execution(execution) == execution + assert execution["attempt_count"] == attempts and execution["retry_count"] == max(0, attempts - 1) + assert execution["outcome"] == outcome + + class SynchronousDecisionEvaluator(ModelEvaluatorBase): _supports_decision_models = True @@ -120,7 +131,9 @@ def record(session, *args, **kwargs): assert len(observed) == 1 and 0 < observed[0] <= bound + 1e-8 if timeout_seconds is None or timeout_seconds > timeout: assert "timed out" in str(raised.value) - assert raised.value.properties == {"decision_model": {"deployment": "Decision-1"}} + assert_failure_execution( + raised.value, 1, "deadline" if timeout_seconds is None or timeout_seconds > timeout else "error" + ) def test_public_synchronous_authentication_consumes_transport_deadline(localhost, monkeypatch): @@ -190,7 +203,7 @@ def record(session, *args, **kwargs): assert len(calls) == len(observed) == 2 assert 0 < observed[1] <= observed[0] - 0.03 assert observed[0] <= 0.8 + 1e-8 - assert raised.value.properties == {"decision_model": {"deployment": "Decision-1"}} + assert_failure_execution(raised.value, 2, "deadline") def test_default_transport_sends_policy_request_and_retries(localhost, monkeypatch): @@ -226,7 +239,7 @@ def test_default_transport_read_timeout_is_enforced(localhost): invoke(endpoint, timeout_seconds=0.05, max_retries=0) assert time.monotonic() - started < 0.4 assert len(calls) == 1 - assert raised.value.properties == {"decision_model": {"deployment": "Decision-1"}} + assert_failure_execution(raised.value, 1, "error") def test_default_transport_native_scalar_timeout_preserves_numeric_retry_deadline(localhost, monkeypatch): @@ -335,7 +348,7 @@ async def request(): await runner(state={}, questions=QUESTIONS, timeout=3) assert time.monotonic() - started < 0.4 assert raised.value.blame == ErrorBlame.USER_ERROR - assert raised.value.properties == {"decision_model": {"deployment": "Decision-1"}} + assert_failure_execution(raised.value, 1, "cancelled") finally: handle.cancel() @@ -437,7 +450,7 @@ async def close(session): else: with pytest.raises(EvaluationException, match="cancelled" if cancel else "timed out") as raised: evaluator(query="synthetic") - assert raised.value.properties == {"decision_model": {"deployment": "Decision-1"}} + assert_failure_execution(raised.value, 1, "cancelled" if cancel else "deadline") elapsed = time.monotonic() - started # The existing batch engine polls run completion every second; the HTTP read must stop sooner. assert elapsed < (1.5 if batch else 0.8) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py index ec8e64b25667..9ec5ac0b4c6b 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py @@ -7,8 +7,11 @@ import asyncio import hashlib import json +import logging import math +import sys import threading +from logging.handlers import BufferingHandler from concurrent.futures import ThreadPoolExecutor from types import SimpleNamespace from unittest.mock import AsyncMock, Mock, patch @@ -34,6 +37,7 @@ validate_score_answer, ) from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException +from azure.ai.evaluation._evaluators._common._decision_execution import DecisionExecution, validated_execution pytestmark = pytest.mark.unittest @@ -420,6 +424,9 @@ def test_verified_http_error_code_preserves_classification_and_accepted_metadata assert raised.value.blame == (ErrorBlame.USER_ERROR if status == 404 else ErrorBlame.SYSTEM_ERROR) assert raised.value.category == ErrorCategory.FAILED_EXECUTION assert raised.value.target == ErrorTarget.EVALUATE + execution = raised.value.properties["decision_model"].pop("execution") + assert validated_execution(execution) == execution + assert execution["outcome"] == "error" assert raised.value.properties == ( { "decision_model": {"deployment": "Decision-1"}, @@ -541,6 +548,32 @@ def test_score_accepts_rounded_distribution_without_normalizing(self): levels = [0.3333333] * 3 assert validate_score_answer(score_answer(levels), 3) == levels + @pytest.mark.parametrize( + "levels", + [ + [0.19, 0.34, 0.11, 0.07, 0.28], + [0.20, 0.34, 0.11, 0.07, 0.29], + [0.19, 0.34, 0.11, 0.07, 0.27], + [0.21, 0.34, 0.11, 0.07, 0.29], + ], + ) + def test_score_accepts_hundredths_rounding_without_normalizing(self, levels): + assert validate_score_answer(score_answer(levels), 5) == levels + + @pytest.mark.parametrize( + "levels", + [ + [0.1] * 5, + [0.22, 0.34, 0.11, 0.07, 0.29], + [0.18, 0.34, 0.11, 0.07, 0.27], + [0.191, 0.339, 0.11, 0.07, 0.28], + [0.201, 0.339, 0.11, 0.07, 0.29], + ], + ) + def test_score_rejects_outside_hundredths_or_finer_precision_tolerance(self, levels): + with pytest.raises(ValueError, match="sum to one"): + validate_score_answer(score_answer(levels), 5) + @pytest.mark.parametrize("levels", [[0] * 5, [0.1] * 5, [0.3] * 5]) def test_score_rejects_nonunit_distribution(self, levels): with pytest.raises(ValueError, match="sum to one"): @@ -549,7 +582,7 @@ def test_score_rejects_nonunit_distribution(self, levels): @pytest.mark.parametrize("direction", [-1, 1]) @pytest.mark.parametrize("inside", [False, True]) def test_score_sum_tolerance_boundary(self, direction, inside): - boundary = 1 + direction * 1e-6 + boundary = 1 + direction * 0.001 total = math.nextafter(boundary, 1.0 if inside else direction * math.inf) levels = [0.5, total - 0.5] assert math.fsum(levels) == total @@ -1143,6 +1176,9 @@ def test_final_http_error_keeps_classification_and_only_accepted_known_usage(sel with pytest.raises(EvaluationException, match=f"HTTP {status}") as raised: run(DecisionModelRunner(DECISION_CONFIG, max_retries=0, max_response_bytes=response_limit)) assert raised.value.blame == (ErrorBlame.USER_ERROR if status == 400 else ErrorBlame.SYSTEM_ERROR) + execution = raised.value.properties["decision_model"].pop("execution") + assert validated_execution(execution) == execution + assert execution["outcome"] == "error" if response_limit == 1: assert raised.value.properties == {"decision_model": {"deployment": "Decision-1"}} else: @@ -1249,12 +1285,14 @@ def test_unknown_or_invalid_usage_is_not_fabricated(self, client): client.response = FakeResponse(body=json.dumps({"usage": {"input_tokens": True, "output_tokens": 3}}).encode()) with pytest.raises(EvaluationException) as raised: run(DecisionModelRunner(DECISION_CONFIG)) + execution = raised.value.properties["decision_model"].pop("execution") + assert execution["usage_state"] == "unknown" assert raised.value.properties == {"decision_model": {"deployment": "Decision-1"}, "completion_tokens": 3} - def test_default_pipeline_has_no_raising_observer_hook(self, client): + def test_default_pipeline_owns_an_execution_observer(self, client): client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) run(DecisionModelRunner(DECISION_CONFIG)) - assert "custom_hook_policy" not in client.policies + assert isinstance(client.policies["custom_hook_policy"], DecisionExecution) assert type(client.policies["retry_policy"]) is AsyncRetryPolicy def test_base_timeout_caps_existing_evaluator_timeout(self, client): @@ -1333,6 +1371,9 @@ def test_authentication_failure_is_sanitized_and_has_no_fabricated_usage(self, c credential = Mock(get_token=Mock(side_effect=RuntimeError("credential-secret"))) with pytest.raises(EvaluationException, match="acquire a token") as raised: run(DecisionModelRunner(config, credential)) + execution = raised.value.properties["decision_model"].pop("execution") + assert execution["attempt_count"] == 0 + assert execution["usage_state"] == "unknown" assert raised.value.properties == {"decision_model": {"deployment": "Decision-1"}} assert "secret" not in str(raised.value) assert not client.calls @@ -1467,6 +1508,9 @@ def cancel(): assert raised.value.blame == ( ErrorBlame.SYSTEM_ERROR if termination == "timeout" else ErrorBlame.USER_ERROR ) + execution = raised.value.properties["decision_model"].pop("execution") + assert execution["attempt_count"] == 1 + assert execution["outcome"] == ("deadline" if termination == "timeout" else "cancelled") assert raised.value.properties == {"decision_model": {"deployment": "Decision-1"}} finally: handle.cancel() @@ -1502,6 +1546,10 @@ async def send(self, request, **kwargs): run(DecisionModelRunner(DECISION_CONFIG, max_retries=max_retries)) expected = max_retries + 1 assert transport.calls == expected + execution = raised.value.properties["decision_model"].pop("execution") + assert execution["attempt_count"] == expected + assert execution["retry_count"] == max_retries + assert execution["outcome"] == "error" assert raised.value.properties == {"decision_model": {"deployment": "Decision-1"}} assert "secret" not in str(raised.value) @@ -1537,3 +1585,217 @@ async def send(self, request, **kwargs): monkeypatch.setattr(_model_runners, "get_async_http_client", lambda **_: AsyncHttpPipeline(transport=transport)) assert run(DecisionModelRunner(DECISION_CONFIG))["usage"] == {"input_tokens": 120, "output_tokens": 3} assert transport.calls == 1 + + +class TestDecisionExecutionObservations: + @pytest.mark.parametrize("case", ["completed", "retry", "error", "deadline", "cancelled", "external_cancelled"]) + @pytest.mark.parametrize("sink", ["normal", "info_off", "log_failure", "metric_failure"]) + def test_actual_pipeline_observations_are_owned_once_and_sinks_are_independent(self, monkeypatch, case, sink): + from azure.ai.evaluation._http_utils import AsyncHttpPipeline + + logger = logging.Logger("decision-test", logging.WARNING if sink == "info_off" else logging.INFO) + handler = BufferingHandler(20) + logger.addHandler(handler) + adapter = logging.LoggerAdapter(logger, {"run_id": "private-run", "correlation_id": "private-correlation"}) + instruments = {} + calls = [] + + def instrument(name, **_): + value = instruments.setdefault(name, Mock()) + if sink == "metric_failure": + value.add.side_effect = RuntimeError("metric-secret") + value.record.side_effect = RuntimeError("metric-secret") + return value + + meter = SimpleNamespace(create_counter=instrument, create_histogram=instrument) + + class Response(HttpResponse): + def __init__(self, request, status, data): + super().__init__(request, None) + self.status_code = status + self.headers = {"Retry-After": "0"} + self.data = data + + def body(self): + return json.dumps(self.data).encode() + + class Transport(AsyncHttpTransport): + async def open(self): + pass + + async def close(self): + pass + + async def __aexit__(self, *_): + await self.close() + + async def send(self, request, **_): + calls.append(request) + if case in ("deadline", "cancelled", "external_cancelled"): + await asyncio.sleep(10) + if case == "error" or (case == "retry" and len(calls) == 1): + return Response(request, 500, {"error": {"message": "provider-secret"}}) + return Response(request, 200, level_payload(4)) + + monkeypatch.setattr( + _model_runners, "get_async_http_client", lambda **_: AsyncHttpPipeline(transport=Transport()) + ) + if sink == "log_failure": + monkeypatch.setattr(logger, "info", Mock(side_effect=RuntimeError("logger-secret"))) + + async def request(): + cancelled = asyncio.Event() + runner = DecisionModelRunner( + DECISION_CONFIG, + max_retries=1 if case == "retry" else 0, + cancellation_requested=cancelled.is_set if case == "cancelled" else None, + ) + runner._bind_execution_logger(adapter) + task = asyncio.current_task() + handle = asyncio.get_running_loop().call_later( + 0.02, task.cancel if case == "external_cancelled" else cancelled.set + ) + try: + if case in ("completed", "retry"): + response = await runner(state="private-evidence", questions=QUESTIONS, timeout=1) + summary = response["_decision_execution"] + assert response["usage"]["input_tokens"] == 120 + else: + with pytest.raises( + asyncio.CancelledError if case == "external_cancelled" else EvaluationException + ) as error: + await runner( + state="private-evidence", questions=QUESTIONS, timeout=0.04 if case == "deadline" else 1 + ) + summary = error.value.properties["decision_model"]["execution"] + assert validated_execution(summary) == summary + assert summary["attempt_count"] == (2 if case == "retry" else 1) + assert summary["retry_count"] == (1 if case == "retry" else 0) + assert summary["outcome"] == ( + "completed" if case == "retry" else "cancelled" if case == "external_cancelled" else case + ) + assert summary["usage_state"] == ( + "known" if case == "completed" else "partial" if case == "retry" else "unknown" + ) + assert summary["last_http_status"] == ( + 200 if case in ("completed", "retry") else 500 if case == "error" else None + ) + finally: + handle.cancel() + + with patch.dict(sys.modules, {"opentelemetry.metrics": SimpleNamespace(get_meter=lambda _: meter)}): + asyncio.run(request()) + expected = 2 if case == "retry" else 1 + assert len(calls) == expected + if sink not in ("info_off", "log_failure"): + assert len(handler.buffer) == expected + 1 + dimensions = [json.loads(record.decision_dimensions) for record in handler.buffer] + assert [item["phase"] for item in dimensions] == ["attempt"] * expected + ["execution"] + for record in handler.buffer: + assert record.run_id == "private-run" and record.correlation_id == "private-correlation" + assert "private-evidence" not in record.decision_execution + record.getMessage() + assert "secret" not in record.decision_execution + record.getMessage() + else: + assert not handler.buffer + if sink != "metric_failure": + prefix = "azure.ai.evaluation.decision." + assert instruments[prefix + "provider.attempts"].add.call_count == expected + assert instruments[prefix + "provider.duration"].record.call_count == expected + instruments[prefix + "operations"].add.assert_called_once() + assert instruments[prefix + "provider.retries"].add.call_count == expected - 1 + for value in instruments.values(): + for call in value.add.call_args_list + value.record.call_args_list: + attributes = call.kwargs["attributes"] + assert set(attributes) == { + "backend", + "route", + "phase", + "outcome", + "usage_state", + "http_status_class", + "throttled", + } + assert "private" not in str(attributes) and "Decision-1" not in str(attributes) + handler.close() + + @pytest.mark.parametrize("outcome", ["completed", "skipped", "semantic_error"]) + def test_strategy_projects_valid_execution_without_reemitting(self, outcome): + response = level_payload(4, status=0.1 if outcome == "skipped" else 0.9) + summary = { + "attempt_count": 1, + "retry_count": 0, + "elapsed_ms": 12.5, + "outcome": "completed", + "last_http_status": 200, + "usage_state": "known", + } + response["_decision_execution"] = summary + if outcome == "semantic_error": + response["answers"]["tool_call_accuracy"]["probabilities"]["4"] = 0.5 + strategy = DecisionEvaluationStrategy( + DecisionModelRunner(DECISION_CONFIG), + result_key="tool_call_accuracy", + error_target=ErrorTarget.EVALUATE, + questions=QUESTIONS, + scoring=LikertScoring(5), + threshold=3, + timeout=1, + ) + with patch.object(DecisionExecution, "emit") as emit: + if outcome == "semantic_error": + with pytest.raises(EvaluationException) as error: + strategy.parse(response, {}) + properties = error.value.properties + else: + properties = strategy.parse(response, {})["tool_call_accuracy_properties"] + assert properties["decision_model"]["execution"] == summary + assert properties["decision_model"]["execution"] is not summary + emit.assert_not_called() + + @pytest.mark.parametrize( + "changes", + [ + {"attempt_count": True}, + {"attempt_count": 1.0}, + {"attempt_count": 22}, + {"retry_count": 21}, + {"elapsed_ms": True}, + {"elapsed_ms": float("nan")}, + {"elapsed_ms": float("inf")}, + {"elapsed_ms": 86400001}, + {"elapsed_ms": 10**1000}, + {"last_http_status": True}, + {"last_http_status": 99}, + {"last_http_status": 600}, + {"outcome": "private-outcome"}, + {"usage_state": "private-usage"}, + ], + ) + def test_malformed_shared_carrier_is_rejected_without_coercion(self, changes): + summary = { + "attempt_count": 1, + "retry_count": 0, + "elapsed_ms": 1, + "outcome": "completed", + "last_http_status": 200, + "usage_state": "known", + } + assert validated_execution({**summary, **changes}) is None + + def test_hosted_summary_bounds_do_not_restrict_public_retry_configuration(self, caplog): + runner = DecisionModelRunner(DECISION_CONFIG, max_retries=100) + assert runner._max_retries == 100 + execution = DecisionExecution(None, None) + execution.attempt_count = 21 + execution.known_usage_attempts = 21 + assert validated_execution(execution.summary("completed"))["retry_count"] == 20 + execution.attempt_count = 22 + with patch.object(DecisionExecution, "emit"): + assert execution.finish("completed") is None + assert "summary omitted" in caplog.text + + def test_optional_metrics_absence_does_not_change_summary_or_require_a_logger(self): + execution = DecisionExecution(None, None) + with patch.dict(sys.modules, {"opentelemetry.metrics": None}): + summary = execution.finish("error") + assert summary["attempt_count"] == 0 and summary["usage_state"] == "unknown" From 1156379b5a79cdb6dd6f50e761d4985d4892ce40 Mon Sep 17 00:00:00 2001 From: salma-elshafey Date: Sun, 18 Oct 2026 00:52:23 +0300 Subject: [PATCH 21/24] Simplify Decision execution and surface provider error codes Consolidate execution tracking into the model runner and expose bounded provider error codes without response messages or details. Simplify test setup while preserving coverage and recognize genuine localhost connection timeouts. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e564c5c8-480e-48e7-a9e0-44450f774b5e --- .../azure-ai-evaluation/CHANGELOG.md | 1 + .../_batch_run/_run_submitter_client.py | 2 +- .../_common/_decision_execution.py | 228 ------------------ .../_evaluators/_common/_decision_strategy.py | 3 +- .../_evaluators/_common/_model_runners.py | 224 ++++++++++++++++- .../test_decision_batch_persistence.py | 65 ++--- .../test_decision_default_transport.py | 12 +- .../unittests/test_decision_model_support.py | 135 +++++------ 8 files changed, 325 insertions(+), 345 deletions(-) delete mode 100644 sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_execution.py diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index ef1795755b10..11e36ad24c94 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -9,6 +9,7 @@ - Decision score probabilities use precision-aware sum validation: hundredth-precision values allow `0.005` rounding error per level plus `1e-12`; other values use an absolute `0.001` tolerance. Invalid distributions raise a system-classified `EvaluationException`; probabilities are not normalized. - Decision results expose known final-response usage, model, and provenance on completed, skipped, and failed evaluations, counting shared requests once and summing split-turn usage. Usage excludes earlier retry attempts. - Decision requests provide context-aware per-attempt and terminal execution observations with optional OpenTelemetry metrics. Validated shared and independent execution summaries and their distinct usage scopes are preserved in failed-row metadata. +- Decision HTTP errors include the provider's nested `error.code` when it starts with an ASCII letter and contains only letters, digits, dots, underscores, or hyphens, up to 128 characters. This is provider-controlled diagnostic text, not a guarantee that the code is free of sensitive data. Messages, details, and raw response bodies are not exposed. - Decision native async HTTP supports overall deadlines and cancellation during response body reads. TLS uses Python/OpenSSL trust settings such as `SSL_CERT_FILE`, not `REQUESTS_CA_BUNDLE`; shared Prompty transport behavior is unchanged. ### Breaking Changes diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py index bd3226987c2a..2a211c59e5e6 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py @@ -23,7 +23,7 @@ from ..._evaluate._utils import _has_aggregator from ..._constants import Prefixes, PF_BATCH_TIMEOUT_SEC from ..._exceptions import EvaluationException -from ..._evaluators._common._decision_execution import validated_execution +from ..._evaluators._common._model_runners import validated_execution from .._utils import get_int_env_var as get_int diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_execution.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_execution.py deleted file mode 100644 index a75005eb73c2..000000000000 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_execution.py +++ /dev/null @@ -1,228 +0,0 @@ -# --------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# --------------------------------------------------------- - -"""Bounded Decision execution metadata and request-owned observations.""" - -import json -import logging -import math -import re -import time -from typing import Any, Dict, Optional - -from azure.core.pipeline.policies import CustomHookPolicy - -LOGGER = logging.getLogger(__name__) -_PREFIX = "azure.ai.evaluation.decision." - - -def _integer(value: Any, maximum: Optional[int] = None) -> bool: - return ( - isinstance(value, int) and not isinstance(value, bool) and 0 <= value and (maximum is None or value <= maximum) - ) - - -def validated_execution(value: Any) -> Optional[Dict[str, Any]]: - """Copy only a complete, strictly typed shared or independent execution summary. - - :param value: The untrusted summary carrier. - :type value: Any - :return: Known validated fields, or None for an invalid carrier. - :rtype: Optional[Dict[str, Any]] - """ - if not isinstance(value, dict): - return None - if "operation_id" in value: - keys = { - "operation_id", - "attempt_count", - "unknown_usage_attempt_count", - "result_usage_scope", - "usage_completeness", - } - if set(value) != keys or not all( - ( - isinstance(value["operation_id"], str) - and re.fullmatch("[0-9a-f]{32}", value["operation_id"]) is not None, - _integer(value["attempt_count"]), - _integer(value["unknown_usage_attempt_count"]), - value["result_usage_scope"] == "final_response_only", - value["usage_completeness"] - in ("all_attempts_reported", "some_attempts_reported", "no_attempts_reported"), - ) - ): - return None - return dict(value) - keys = {"attempt_count", "retry_count", "elapsed_ms", "outcome", "last_http_status", "usage_state"} - elapsed = value.get("elapsed_ms") - status = value.get("last_http_status") - if ( - not keys - {"last_http_status"} <= value.keys() - or isinstance(elapsed, bool) - or not isinstance(elapsed, (int, float)) - ): - return None - if not all( - ( - _integer(value["attempt_count"], 21), - _integer(value["retry_count"], 20), - 0 <= elapsed <= 86400000 and math.isfinite(elapsed), - value["outcome"] in ("completed", "error", "deadline", "cancelled"), - value["usage_state"] in ("known", "partial", "unknown"), - status is None or (_integer(status, 599) and status >= 100), - ) - ): - return None - return {key: value.get(key) for key in keys} - - -def _diagnostic(message: str) -> None: - # Observer failures must not replace a model result, even if diagnostic logging is also broken. - try: - LOGGER.warning(message) - except Exception: # pylint: disable=broad-except - pass - - -class DecisionExecution(CustomHookPolicy): - """One operation's per-attempt pipeline policy; never shared between concurrent calls.""" - - def __init__(self, logger: Any, max_response_bytes: Optional[int]) -> None: - super().__init__() - self.logger = logger - self.max_response_bytes = max_response_bytes - self.started = time.monotonic() - self.attempt_started: Optional[float] = None - self.attempt_count = 0 - self.known_usage_attempts = 0 - self.partial_usage_attempts = 0 - self.last_http_status: Optional[int] = None - self.outcome = "error" - self.instruments: Optional[Dict[str, Any]] = None - - def on_request(self, request: Any) -> None: - self.attempt_count += 1 - self.attempt_started = time.monotonic() - super().on_request(request) - - def on_response(self, request: Any, response: Any) -> None: - raw = response.http_response - status = raw.status_code - self.last_http_status = status if _integer(status, 599) and status >= 100 else None - known = False - partial = False - try: - body = raw.body() - if self.max_response_bytes is None or len(body) <= self.max_response_bytes: - payload = json.loads(body) - usage = payload.get("usage") if isinstance(payload, dict) else None - if isinstance(usage, dict): - tokens = [_integer(usage.get(key)) for key in ("input_tokens", "output_tokens")] - known, partial = all(tokens), any(tokens) - except (ValueError, TypeError): - pass - self.known_usage_attempts += int(known) - self.partial_usage_attempts += int(partial and not known) - self.finish_attempt( - "completed" if status == 200 else "error", - "known" if known else "partial" if partial else "unknown", - self.last_http_status, - ) - super().on_response(request, response) - - def on_exception(self, request: Any) -> None: - self.finish_attempt("error") - - def summary(self, outcome: str) -> Dict[str, Any]: - return { - "attempt_count": self.attempt_count, - "retry_count": max(0, self.attempt_count - 1), - "elapsed_ms": (time.monotonic() - self.started) * 1000, - "outcome": outcome, - "last_http_status": self.last_http_status, - "usage_state": ( - "known" - if self.attempt_count and self.known_usage_attempts == self.attempt_count - else "partial" if self.known_usage_attempts or self.partial_usage_attempts else "unknown" - ), - } - - def finish_attempt(self, outcome: str, usage: str = "unknown", status: Optional[int] = None) -> None: - if self.attempt_started is None: - return - elapsed = (time.monotonic() - self.attempt_started) * 1000 - self.attempt_started = None - self.emit("attempt", outcome, self.summary(outcome), elapsed, usage, status) - - def finish(self, outcome: str) -> Optional[Dict[str, Any]]: - self.finish_attempt(outcome) - summary = self.summary(outcome) - self.emit("execution", outcome, summary, summary["elapsed_ms"], summary["usage_state"]) - accepted = validated_execution(summary) - if accepted is None: - _diagnostic("Decision execution summary exceeds the hosted metadata bounds; summary omitted.") - return accepted - - def emit( - self, - phase: str, - outcome: str, - summary: Dict[str, Any], - elapsed: float, - usage: str, - status: Optional[int] = None, - ) -> None: - status = self.last_http_status if phase == "execution" else status - dimensions = { - "backend": "decision", - "route": "shared_sdk", - "phase": phase, - "outcome": outcome, - "usage_state": usage, - "http_status_class": f"{status // 100}xx" if status is not None else "unknown", - "throttled": str(status == 429).lower(), - } - try: - logger = self.logger - if logger is not None and logger.isEnabledFor(logging.INFO): - fields = { - "decision_dimensions": json.dumps(dimensions), - "decision_execution": json.dumps(summary), - } - while isinstance(logger, logging.LoggerAdapter): - # Python 3.10 LoggerAdapter.process replaces, rather than merges, per-call extra fields. - _, kwargs = logger.process("DECISION_EXECUTION_OBSERVATION", {"extra": dict(fields)}) - fields = {**kwargs.get("extra", {}), **fields} - logger = logger.logger - logger.info("DECISION_EXECUTION_OBSERVATION", extra=fields) - except Exception: # pylint: disable=broad-except - _diagnostic("Decision execution log sink failed; evaluation and metrics continue.") - try: - self.emit_metrics(phase, elapsed, dimensions) - except Exception: # pylint: disable=broad-except - _diagnostic("Decision execution metric sink failed; evaluation and logs continue.") - - def emit_metrics(self, phase: str, elapsed: float, dimensions: Dict[str, str]) -> None: - if self.instruments is None: - try: - from opentelemetry.metrics import get_meter # pylint: disable=import-outside-toplevel - except ImportError: - self.instruments = {} - LOGGER.debug("Decision metrics require the optional opentelemetry extra.") - return - meter = get_meter("azure.ai.evaluation.decision") - self.instruments = { - key: meter.create_counter(_PREFIX + key) - for key in ("provider.attempts", "provider.retries", "operations") - } - self.instruments["provider.duration"] = meter.create_histogram(_PREFIX + "provider.duration", unit="ms") - if not self.instruments: - return - if phase == "attempt": - self.instruments["provider.attempts"].add(1, attributes=dimensions) - self.instruments["provider.duration"].record(elapsed, attributes=dimensions) - if self.attempt_count > 1: - self.instruments["provider.retries"].add(1, attributes=dimensions) - else: - self.instruments["operations"].add(1, attributes=dimensions) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py index 457a2f0723b8..cb5aaee4a7f3 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py @@ -15,8 +15,7 @@ from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException from ._base_model_eval import not_applicable_result -from ._model_runners import DecisionModelRunner, validate_noul_answer, validate_score_answer -from ._decision_execution import validated_execution +from ._model_runners import DecisionModelRunner, validate_noul_answer, validate_score_answer, validated_execution STATUS_CUTOFF = 0.5 T = TypeVar("T") diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py index 8b5050fc76b6..053fecdc5d45 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_model_runners.py @@ -15,17 +15,17 @@ import logging import math import os +import re import time from typing import Any, Callable, Dict, List, Optional from urllib.parse import urlsplit -from azure.core.pipeline.policies import AsyncRedirectPolicy, AsyncRetryPolicy +from azure.core.pipeline.policies import AsyncRedirectPolicy, AsyncRetryPolicy, CustomHookPolicy from azure.core.pipeline.transport import AioHttpTransport from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException from ..._common.utils import construct_prompty_model_config, validate_model_config from ..._http_utils import get_async_http_client -from ._decision_execution import DecisionExecution if os.getenv("AI_EVALS_USE_PF_PROMPTY", "false").lower() == "true": from promptflow.core._flow import AsyncPrompty @@ -46,7 +46,8 @@ def value(self) -> str: _DECISION_PATH = "/providers/microsoft/v1/systemone" _DECISION_TOKEN_SCOPE = "https://cognitiveservices.azure.com/.default" _DECISION_RETRY_STATUS_CODES = [408, 429, 500, 502, 503, 504, 529] -_DECISION_SAFE_ERROR_CODES = {"DeploymentNotFound"} +LOGGER = logging.getLogger(__name__) +_PREFIX = "azure.ai.evaluation.decision." _DECISION_CONFIG_KEYS = { "type", "azure_endpoint", @@ -71,6 +72,217 @@ def is_decision_model_config(model_config: Any) -> bool: return isinstance(model_config, dict) and model_config.get("type") == DECISION_MODEL_TYPE +def _integer(value: Any, maximum: Optional[int] = None) -> bool: + return ( + isinstance(value, int) and not isinstance(value, bool) and 0 <= value and (maximum is None or value <= maximum) + ) + + +def validated_execution(value: Any) -> Optional[Dict[str, Any]]: + """Copy only a complete, strictly typed shared or independent execution summary. + + :param value: The untrusted summary carrier. + :type value: Any + :return: Known validated fields, or None for an invalid carrier. + :rtype: Optional[Dict[str, Any]] + """ + if not isinstance(value, dict): + return None + if "operation_id" in value: + keys = { + "operation_id", + "attempt_count", + "unknown_usage_attempt_count", + "result_usage_scope", + "usage_completeness", + } + if set(value) != keys or not all( + ( + isinstance(value["operation_id"], str) + and re.fullmatch("[0-9a-f]{32}", value["operation_id"]) is not None, + _integer(value["attempt_count"]), + _integer(value["unknown_usage_attempt_count"]), + value["result_usage_scope"] == "final_response_only", + value["usage_completeness"] + in ("all_attempts_reported", "some_attempts_reported", "no_attempts_reported"), + ) + ): + return None + return dict(value) + keys = {"attempt_count", "retry_count", "elapsed_ms", "outcome", "last_http_status", "usage_state"} + elapsed = value.get("elapsed_ms") + status = value.get("last_http_status") + if ( + not keys - {"last_http_status"} <= value.keys() + or isinstance(elapsed, bool) + or not isinstance(elapsed, (int, float)) + ): + return None + if not all( + ( + _integer(value["attempt_count"], 21), + _integer(value["retry_count"], 20), + 0 <= elapsed <= 86400000 and math.isfinite(elapsed), + value["outcome"] in ("completed", "error", "deadline", "cancelled"), + value["usage_state"] in ("known", "partial", "unknown"), + status is None or (_integer(status, 599) and status >= 100), + ) + ): + return None + return {key: value.get(key) for key in keys} + + +def _diagnostic(message: str) -> None: + # Observer failures must not replace a model result, even if diagnostic logging is also broken. + try: + LOGGER.warning(message) + except Exception: # pylint: disable=broad-except + pass + + +class DecisionExecution(CustomHookPolicy): + """One operation's per-attempt pipeline policy; never shared between concurrent calls.""" + + def __init__(self, logger: Any, max_response_bytes: Optional[int]) -> None: + super().__init__() + self.logger = logger + self.max_response_bytes = max_response_bytes + self.started = time.monotonic() + self.attempt_started: Optional[float] = None + self.attempt_count = 0 + self.known_usage_attempts = 0 + self.partial_usage_attempts = 0 + self.last_http_status: Optional[int] = None + self.outcome = "error" + self.instruments: Optional[Dict[str, Any]] = None + + def on_request(self, request: Any) -> None: + self.attempt_count += 1 + self.attempt_started = time.monotonic() + super().on_request(request) + + def on_response(self, request: Any, response: Any) -> None: + raw = response.http_response + status = raw.status_code + self.last_http_status = status if _integer(status, 599) and status >= 100 else None + known = False + partial = False + try: + body = raw.body() + if self.max_response_bytes is None or len(body) <= self.max_response_bytes: + payload = json.loads(body) + usage = payload.get("usage") if isinstance(payload, dict) else None + if isinstance(usage, dict): + tokens = [_integer(usage.get(key)) for key in ("input_tokens", "output_tokens")] + known, partial = all(tokens), any(tokens) + except (ValueError, TypeError): + pass + self.known_usage_attempts += int(known) + self.partial_usage_attempts += int(partial and not known) + self.finish_attempt( + "completed" if status == 200 else "error", + "known" if known else "partial" if partial else "unknown", + self.last_http_status, + ) + super().on_response(request, response) + + def on_exception(self, request: Any) -> None: + self.finish_attempt("error") + + def summary(self, outcome: str) -> Dict[str, Any]: + return { + "attempt_count": self.attempt_count, + "retry_count": max(0, self.attempt_count - 1), + "elapsed_ms": (time.monotonic() - self.started) * 1000, + "outcome": outcome, + "last_http_status": self.last_http_status, + "usage_state": ( + "known" + if self.attempt_count and self.known_usage_attempts == self.attempt_count + else "partial" if self.known_usage_attempts or self.partial_usage_attempts else "unknown" + ), + } + + def finish_attempt(self, outcome: str, usage: str = "unknown", status: Optional[int] = None) -> None: + if self.attempt_started is None: + return + elapsed = (time.monotonic() - self.attempt_started) * 1000 + self.attempt_started = None + self.emit("attempt", outcome, self.summary(outcome), elapsed, usage, status) + + def finish(self, outcome: str) -> Optional[Dict[str, Any]]: + self.finish_attempt(outcome) + summary = self.summary(outcome) + self.emit("execution", outcome, summary, summary["elapsed_ms"], summary["usage_state"]) + accepted = validated_execution(summary) + if accepted is None: + _diagnostic("Decision execution summary exceeds the hosted metadata bounds; summary omitted.") + return accepted + + def emit( + self, + phase: str, + outcome: str, + summary: Dict[str, Any], + elapsed: float, + usage: str, + status: Optional[int] = None, + ) -> None: + status = self.last_http_status if phase == "execution" else status + dimensions = { + "backend": "decision", + "route": "shared_sdk", + "phase": phase, + "outcome": outcome, + "usage_state": usage, + "http_status_class": f"{status // 100}xx" if status is not None else "unknown", + "throttled": str(status == 429).lower(), + } + try: + logger = self.logger + if logger is not None and logger.isEnabledFor(logging.INFO): + fields = { + "decision_dimensions": json.dumps(dimensions), + "decision_execution": json.dumps(summary), + } + while isinstance(logger, logging.LoggerAdapter): + # Python 3.10 LoggerAdapter.process replaces, rather than merges, per-call extra fields. + _, kwargs = logger.process("DECISION_EXECUTION_OBSERVATION", {"extra": dict(fields)}) + fields = {**kwargs.get("extra", {}), **fields} + logger = logger.logger + logger.info("DECISION_EXECUTION_OBSERVATION", extra=fields) + except Exception: # pylint: disable=broad-except + _diagnostic("Decision execution log sink failed; evaluation and metrics continue.") + try: + self.emit_metrics(phase, elapsed, dimensions) + except Exception: # pylint: disable=broad-except + _diagnostic("Decision execution metric sink failed; evaluation and logs continue.") + + def emit_metrics(self, phase: str, elapsed: float, dimensions: Dict[str, str]) -> None: + if self.instruments is None: + try: + from opentelemetry.metrics import get_meter # pylint: disable=import-outside-toplevel + except ImportError: + self.instruments = {} + LOGGER.debug("Decision metrics require the optional opentelemetry extra.") + return + meter = get_meter("azure.ai.evaluation.decision") + self.instruments = { + key: meter.create_counter(_PREFIX + key) + for key in ("provider.attempts", "provider.retries", "operations") + } + self.instruments["provider.duration"] = meter.create_histogram(_PREFIX + "provider.duration", unit="ms") + if not self.instruments: + return + if phase == "attempt": + self.instruments["provider.attempts"].add(1, attributes=dimensions) + self.instruments["provider.duration"].record(elapsed, attributes=dimensions) + if self.attempt_count > 1: + self.instruments["provider.retries"].add(1, attributes=dimensions) + else: + self.instruments["operations"].add(1, attributes=dimensions) + + class PromptyModelRunner: """Loads a Prompty flow for a chat model and invokes it without interpreting the output.""" @@ -510,7 +722,11 @@ async def _post(self, content: bytes, *, deadline: float, execution: DecisionExe blame = ErrorBlame.USER_ERROR if status in (400, 401, 403, 404, 422) else ErrorBlame.SYSTEM_ERROR provider_error = payload.get("error") if isinstance(payload, dict) else None code = provider_error.get("code") if isinstance(provider_error, dict) else None - diagnostic = f" ({code})" if isinstance(code, str) and code in _DECISION_SAFE_ERROR_CODES else "" + diagnostic = ( + f" ({code})" + if isinstance(code, str) and re.fullmatch(r"[A-Za-z][A-Za-z0-9_.-]{0,127}", code) is not None + else "" + ) error = _failed(f"The decision model returned HTTP {status}{diagnostic}.", blame) error.properties = self._response_properties(payload) raise error diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py index 54276f05cc3a..787b8c4b092c 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py @@ -108,47 +108,41 @@ def strategy(runner): class TestDecisionBatchPersistence: - @pytest.mark.parametrize("backend", ["system_one", "foundry_decision"]) - @pytest.mark.parametrize("known", [False, True]) - @pytest.mark.parametrize("attempted", [False, True]) + EXECUTION = { + "operation_id": "a" * 32, + "attempt_count": 1, + "unknown_usage_attempt_count": 0, + "result_usage_scope": "final_response_only", + "usage_completeness": "all_attempts_reported", + } + + @pytest.mark.parametrize( + "backend,known,attempted", + [("system_one", False, False), ("foundry_decision", False, True), ("system_one", True, True)], + ) def test_independent_failed_metadata_preserves_distinct_scopes_and_known_fields(self, backend, known, attempted): - native = { + expected = { "backend": backend, "protocol": "system_one", "usage_available": known, "result_usage_scope": "reported_attempts_only", - "private": "never-persist", - } - execution = { - "operation_id": "abcdef0123456789" * 2, - "attempt_count": 1, - "unknown_usage_attempt_count": int(not known), - "result_usage_scope": "final_response_only", - "usage_completeness": "all_attempts_reported" if known else "no_attempts_reported", } if attempted: - native["execution"] = execution - properties = {"decision_model": native, "private": "never-persist"} - if known: - properties.update(prompt_tokens=41, completion_tokens=3, total_tokens=44, model="independent-model") + expected["execution"] = { + **self.EXECUTION, + "unknown_usage_attempt_count": int(not known), + "usage_completeness": "all_attempts_reported" if known else "no_attempts_reported", + } + usage = dict(prompt_tokens=41, completion_tokens=3, total_tokens=44, model="independent-model") if known else {} error = EvaluationException( "synthetic failure", - category=ErrorCategory.FAILED_EXECUTION, - blame=ErrorBlame.SYSTEM_ERROR, - properties=properties, + properties={ + **usage, + "decision_model": {**expected, "private": "never-persist"}, + "private": "never-persist", + }, ) - saved = _decision_error_metadata(error)["properties"] - provenance = saved["decision_model"] - assert provenance["backend"] == backend and provenance["protocol"] == "system_one" - assert provenance["usage_available"] is known - assert provenance["result_usage_scope"] == "reported_attempts_only" - assert "private" not in json.dumps(saved) - if attempted: - assert provenance["execution"] == execution - assert provenance["execution"]["result_usage_scope"] == "final_response_only" - else: - assert "execution" not in provenance - assert saved.get("total_tokens") == (44 if known else None) + assert _decision_error_metadata(error)["properties"] == {**usage, "decision_model": expected} @pytest.mark.parametrize( "changes", @@ -166,16 +160,9 @@ def test_independent_failed_metadata_preserves_distinct_scopes_and_known_fields( ], ) def test_malformed_independent_summary_is_dropped_not_rewritten(self, changes): - execution = { - "operation_id": "a" * 32, - "attempt_count": 1, - "unknown_usage_attempt_count": 0, - "result_usage_scope": "final_response_only", - "usage_completeness": "all_attempts_reported", - } error = EvaluationException( "synthetic failure", - properties={"decision_model": {"execution": {**execution, **changes}, "usage_available": False}}, + properties={"decision_model": {"execution": {**self.EXECUTION, **changes}, "usage_available": False}}, ) assert _decision_error_metadata(error)["properties"]["decision_model"] == {"usage_available": False} diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_default_transport.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_default_transport.py index d7fdb5fd56a5..d8d0a43b9aaf 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_default_transport.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_default_transport.py @@ -15,12 +15,13 @@ import requests from azure.ai.evaluation._evaluators._common._model_runners import DecisionModelRunner -from azure.ai.evaluation._evaluators._common._decision_execution import validated_execution +from azure.ai.evaluation._evaluators._common._model_runners import validated_execution from azure.ai.evaluation._evaluators._common._base_model_eval import ModelEvaluatorBase from azure.ai.evaluation._exceptions import ErrorBlame, EvaluationException from azure.ai.evaluation._http_utils import get_async_http_client from azure.core.pipeline.policies import AsyncRetryPolicy from azure.core.pipeline.transport import AsyncioRequestsTransport +from azure.core.exceptions import ServiceRequestTimeoutError pytestmark = pytest.mark.unittest QUESTIONS = {"status": {"type": "noul", "instructions": "Applicable?"}} @@ -126,7 +127,14 @@ def record(session, *args, **kwargs): elapsed = time.monotonic() - started print(f"Entire public synchronous evaluator call: {elapsed:.3f}s") assert elapsed < 0.4 - assert len(calls) == 1 + if ( + timeout_seconds is not None + and timeout_seconds < timeout + and isinstance(raised.value.__context__, ServiceRequestTimeoutError) + ): + assert not calls + else: + assert len(calls) == 1 bound = min(timeout, timeout_seconds) if timeout_seconds is not None else timeout assert len(observed) == 1 and 0 < observed[0] <= bound + 1e-8 if timeout_seconds is None or timeout_seconds > timeout: diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py index 9ec5ac0b4c6b..b3a59745a18e 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py @@ -37,7 +37,7 @@ validate_score_answer, ) from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException -from azure.ai.evaluation._evaluators._common._decision_execution import DecisionExecution, validated_execution +from azure.ai.evaluation._evaluators._common._model_runners import DecisionExecution, validated_execution pytestmark = pytest.mark.unittest @@ -82,8 +82,9 @@ def level_payload(level, status=0.95): return decision_payload(status, {"tool_call_accuracy": score_answer(levels)}) -class FakeResponse: - def __init__(self, status_code=200, body=b"{}"): +class FakeResponse(HttpResponse): + def __init__(self, status_code=200, body=b"{}", request=None): + super().__init__(request, None) self.status_code = status_code self._body = body @@ -404,21 +405,31 @@ def test_http_failures_are_sanitized(self, client, status, blame): assert str(status) in str(raised.value) assert "test-key-value" not in str(raised.value) and "secret detail" not in str(raised.value) - @pytest.mark.parametrize("status", [404, 500]) - @pytest.mark.parametrize("response_limit", [1, 1024]) - def test_verified_http_error_code_preserves_classification_and_accepted_metadata( - self, client, status, response_limit + @pytest.mark.parametrize( + "status,response_limit,code", + [ + (404, 1024, "DeploymentNotFound"), + (500, 1024, "InvalidRequest"), + (404, 1024, "RateLimitExceeded"), + (404, 1024, "secret_like_identifier_123"), + (500, 1024, "A" * 128), + (404, 1, "DeploymentNotFound"), + (500, 1, "InvalidRequest"), + ], + ) + def test_provider_http_error_code_preserves_classification_and_accepted_metadata( + self, client, status, response_limit, code ): payload = decision_payload() payload["error"] = { - "code": "DeploymentNotFound", + "code": code, "message": "test-key-value and private-message", "details": "private-details", } client.response = FakeResponse(status, json.dumps(payload).encode()) with pytest.raises(EvaluationException) as raised: run(DecisionModelRunner(DECISION_CONFIG, max_retries=0, max_response_bytes=response_limit)) - diagnostic = " (DeploymentNotFound)" if response_limit == 1024 else "" + diagnostic = f" ({code})" if response_limit == 1024 else "" prefix = "UserError" if status == 404 else "InternalError" assert str(raised.value) == f"({prefix}) The decision model returned HTTP {status}{diagnostic}." assert raised.value.blame == (ErrorBlame.USER_ERROR if status == 404 else ErrorBlame.SYSTEM_ERROR) @@ -442,9 +453,9 @@ def test_verified_http_error_code_preserves_classification_and_accepted_metadata @pytest.mark.parametrize( "payload", [ - {"error": {"code": "secret_like_identifier_123"}}, - {"error": {"code": "UnknownProviderCode"}}, - {"error": {"code": "deploymentnotfound"}}, + {"error": {"code": ""}}, + {"error": {"code": "InvalidRequest\nprivate-message"}}, + {"error": {"code": "A" * 129}}, {"error": {"code": "DeploymentNotFound "}}, {"error": {"code": None}}, {"error": {"code": 404}}, @@ -460,14 +471,14 @@ def test_verified_http_error_code_preserves_classification_and_accepted_metadata None, ], ) - def test_unverified_or_malformed_http_error_codes_are_not_exposed(self, client, payload): + def test_malformed_or_missing_nested_http_error_codes_are_not_exposed(self, client, payload): client.response = FakeResponse(404, json.dumps(payload).encode()) with pytest.raises(EvaluationException) as raised: run(DecisionModelRunner(DECISION_CONFIG)) assert str(raised.value) == "(UserError) The decision model returned HTTP 404." assert raised.value.blame == ErrorBlame.USER_ERROR - def test_non_json_http_error_cannot_surface_verified_code_text(self, client): + def test_non_json_http_error_cannot_surface_code_text(self, client): client.response = FakeResponse(404, b"DeploymentNotFound private-message test-key-value") with pytest.raises(EvaluationException) as raised: run(DecisionModelRunner(DECISION_CONFIG)) @@ -563,7 +574,6 @@ def test_score_accepts_hundredths_rounding_without_normalizing(self, levels): @pytest.mark.parametrize( "levels", [ - [0.1] * 5, [0.22, 0.34, 0.11, 0.07, 0.29], [0.18, 0.34, 0.11, 0.07, 0.27], [0.191, 0.339, 0.11, 0.07, 0.28], @@ -1588,8 +1598,29 @@ async def send(self, request, **kwargs): class TestDecisionExecutionObservations: - @pytest.mark.parametrize("case", ["completed", "retry", "error", "deadline", "cancelled", "external_cancelled"]) - @pytest.mark.parametrize("sink", ["normal", "info_off", "log_failure", "metric_failure"]) + SUMMARY = { + "attempt_count": 1, + "retry_count": 0, + "elapsed_ms": 12.5, + "outcome": "completed", + "last_http_status": 200, + "usage_state": "known", + } + + @pytest.mark.parametrize( + "case,sink", + [ + ("completed", "normal"), + ("retry", "normal"), + ("error", "normal"), + ("deadline", "normal"), + ("cancelled", "normal"), + ("external_cancelled", "normal"), + ("completed", "info_off"), + ("completed", "log_failure"), + ("completed", "metric_failure"), + ], + ) def test_actual_pipeline_observations_are_owned_once_and_sinks_are_independent(self, monkeypatch, case, sink): from azure.ai.evaluation._http_utils import AsyncHttpPipeline @@ -1598,7 +1629,6 @@ def test_actual_pipeline_observations_are_owned_once_and_sinks_are_independent(s logger.addHandler(handler) adapter = logging.LoggerAdapter(logger, {"run_id": "private-run", "correlation_id": "private-correlation"}) instruments = {} - calls = [] def instrument(name, **_): value = instruments.setdefault(name, Mock()) @@ -1609,37 +1639,19 @@ def instrument(name, **_): meter = SimpleNamespace(create_counter=instrument, create_histogram=instrument) - class Response(HttpResponse): - def __init__(self, request, status, data): - super().__init__(request, None) - self.status_code = status - self.headers = {"Retry-After": "0"} - self.data = data - - def body(self): - return json.dumps(self.data).encode() - - class Transport(AsyncHttpTransport): - async def open(self): - pass - - async def close(self): - pass - - async def __aexit__(self, *_): - await self.close() + async def send(request, **_): + if case in ("deadline", "cancelled", "external_cancelled"): + await asyncio.sleep(10) + failed = case == "error" or (case == "retry" and transport.send.await_count == 1) + payload = {"error": {"message": "provider-secret"}} if failed else level_payload(4) + response = FakeResponse(500 if failed else 200, json.dumps(payload).encode(), request) + response.headers["Retry-After"] = "0" + return response - async def send(self, request, **_): - calls.append(request) - if case in ("deadline", "cancelled", "external_cancelled"): - await asyncio.sleep(10) - if case == "error" or (case == "retry" and len(calls) == 1): - return Response(request, 500, {"error": {"message": "provider-secret"}}) - return Response(request, 200, level_payload(4)) + transport = AsyncMock(spec=AsyncHttpTransport) + transport.send.side_effect = send - monkeypatch.setattr( - _model_runners, "get_async_http_client", lambda **_: AsyncHttpPipeline(transport=Transport()) - ) + monkeypatch.setattr(_model_runners, "get_async_http_client", lambda **_: AsyncHttpPipeline(transport=transport)) if sink == "log_failure": monkeypatch.setattr(logger, "info", Mock(side_effect=RuntimeError("logger-secret"))) @@ -1656,17 +1668,16 @@ async def request(): 0.02, task.cancel if case == "external_cancelled" else cancelled.set ) try: + call = runner(state="private-evidence", questions=QUESTIONS, timeout=0.04 if case == "deadline" else 1) if case in ("completed", "retry"): - response = await runner(state="private-evidence", questions=QUESTIONS, timeout=1) + response = await call summary = response["_decision_execution"] assert response["usage"]["input_tokens"] == 120 else: with pytest.raises( asyncio.CancelledError if case == "external_cancelled" else EvaluationException ) as error: - await runner( - state="private-evidence", questions=QUESTIONS, timeout=0.04 if case == "deadline" else 1 - ) + await call summary = error.value.properties["decision_model"]["execution"] assert validated_execution(summary) == summary assert summary["attempt_count"] == (2 if case == "retry" else 1) @@ -1686,7 +1697,8 @@ async def request(): with patch.dict(sys.modules, {"opentelemetry.metrics": SimpleNamespace(get_meter=lambda _: meter)}): asyncio.run(request()) expected = 2 if case == "retry" else 1 - assert len(calls) == expected + assert transport.send.await_count == expected + transport.__aexit__.assert_awaited_once() if sink not in ("info_off", "log_failure"): assert len(handler.buffer) == expected + 1 dimensions = [json.loads(record.decision_dimensions) for record in handler.buffer] @@ -1721,14 +1733,7 @@ async def request(): @pytest.mark.parametrize("outcome", ["completed", "skipped", "semantic_error"]) def test_strategy_projects_valid_execution_without_reemitting(self, outcome): response = level_payload(4, status=0.1 if outcome == "skipped" else 0.9) - summary = { - "attempt_count": 1, - "retry_count": 0, - "elapsed_ms": 12.5, - "outcome": "completed", - "last_http_status": 200, - "usage_state": "known", - } + summary = self.SUMMARY response["_decision_execution"] = summary if outcome == "semantic_error": response["answers"]["tool_call_accuracy"]["probabilities"]["4"] = 0.5 @@ -1772,15 +1777,7 @@ def test_strategy_projects_valid_execution_without_reemitting(self, outcome): ], ) def test_malformed_shared_carrier_is_rejected_without_coercion(self, changes): - summary = { - "attempt_count": 1, - "retry_count": 0, - "elapsed_ms": 1, - "outcome": "completed", - "last_http_status": 200, - "usage_state": "known", - } - assert validated_execution({**summary, **changes}) is None + assert validated_execution({**self.SUMMARY, **changes}) is None def test_hosted_summary_bounds_do_not_restrict_public_retry_configuration(self, caplog): runner = DecisionModelRunner(DECISION_CONFIG, max_retries=100) From a30074f3deb0070774252c74aeab3ea83f9fda68 Mon Sep 17 00:00:00 2001 From: salma-elshafey Date: Sun, 18 Oct 2026 01:37:31 +0300 Subject: [PATCH 22/24] Fix Decision serialization test under INFO logging Assert only evidence-bearing request serializations so execution telemetry does not affect the request-count invariant. Enable INFO logging explicitly to cover the CI failure mode. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e564c5c8-480e-48e7-a9e0-44450f774b5e --- .../unittests/test_decision_model_support.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py index b3a59745a18e..d28bd7d6b9fa 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py @@ -14,7 +14,7 @@ from logging.handlers import BufferingHandler from concurrent.futures import ThreadPoolExecutor from types import SimpleNamespace -from unittest.mock import AsyncMock, Mock, patch +from unittest.mock import AsyncMock, Mock, call, patch import pytest from openai.types.chat import ChatCompletion @@ -1257,16 +1257,20 @@ def test_request_budget_exact_boundary_and_no_send_on_overflow(self, client, fie assert raised.value.blame == ErrorBlame.USER_ERROR @pytest.mark.parametrize("request_cap", [None, 10000]) - def test_uncapped_state_question_pairs_are_not_serialized(self, client, request_cap): + def test_uncapped_state_question_pairs_are_not_serialized(self, client, request_cap, caplog): + caplog.set_level(logging.INFO, logger=_model_runners.__name__) client.response = FakeResponse(body=json.dumps(level_payload(4)).encode()) with patch.object(_model_runners.json, "dumps", wraps=json.dumps) as serialize: run(DecisionModelRunner(DECISION_CONFIG, max_request_bytes=request_cap)) - serialize.assert_called_once_with( - {"model": "Decision-1", "state": {"query": "q"}, "questions": QUESTIONS}, - ensure_ascii=False, - separators=(",", ":"), - allow_nan=False, - ) + request_serializations = [entry for entry in serialize.call_args_list if "state" in entry.args[0]] + assert request_serializations == [ + call( + {"model": "Decision-1", "state": {"query": "q"}, "questions": QUESTIONS}, + ensure_ascii=False, + separators=(",", ":"), + allow_nan=False, + ) + ] def test_non_json_request_is_rejected_before_authentication_or_network(self, client): with pytest.raises(EvaluationException, match="JSON-serializable") as raised: From 3fdd873c013578a9c44dbab97475016915392faa Mon Sep 17 00:00:00 2001 From: salma-elshafey Date: Mon, 19 Oct 2026 00:07:58 +0300 Subject: [PATCH 23/24] Preserve Prompty usage on post-inference skips Use existing result formatting to retain measured usage, model metadata, and response properties for skipped model responses while leaving pre-inference skips unchanged. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e564c5c8-480e-48e7-a9e0-44450f774b5e --- .../azure-ai-evaluation/CHANGELOG.md | 1 + .../_evaluators/_common/_prompty_strategy.py | 9 ++- .../tests/unittests/test_prompty_strategy.py | 70 ++++++++++++++++++- 3 files changed, 77 insertions(+), 3 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index 11e36ad24c94..788c45ad08c3 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -16,6 +16,7 @@ ### Bugs Fixed +- Preserved measured token usage, model metadata, and response properties when the shared Prompty strategy parses a post-inference skipped result. Pre-inference skips still make no model request. - Hardened local Prompty image resolution so only relative files within the Prompty directory are inlined, with opened-file validation to prevent path changes from redirecting reads and without exposing the local directory in fallback logs. ### Other Changes diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_prompty_strategy.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_prompty_strategy.py index 9e2516003d53..7f48b718fdbe 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_prompty_strategy.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_prompty_strategy.py @@ -70,7 +70,14 @@ def parse(self, output: Dict[str, Any]) -> Dict[str, Any]: score = self.missing_score if isinstance(answer, dict) and (self.strict or answer): if answer.get("status", "completed") == "skipped": - return self.skipped(answer.get("reason", "")) + return self.format_result( + None, + "not_applicable", + f"Not applicable: {answer.get('reason', '')}", + "skipped", + answer.get("properties", {}) or {}, + output, + ) score = answer.get("score", self.missing_score) reason = answer.get("reason", "") properties = answer.get("properties", {}) or {} diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_strategy.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_strategy.py index 184919e09b38..c676e55290e1 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_strategy.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_prompty_strategy.py @@ -111,7 +111,15 @@ def test_skipped_bypasses_score_validation(strict): "metric_reason": "Not applicable: not relevant", "metric_status": "skipped", "metric_threshold": 3, - "metric_properties": None, + "metric_properties": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "finish_reason": "", + "model": "", + "sample_input": "", + "sample_output": "", + }, } @@ -159,8 +167,66 @@ def test_exact_chat_schemas(key, alias, usage): assert set(completed) == expected_keys skipped = configured.parse({"llm_output": {"status": "skipped"}, "input_token_count": 9}) assert set(skipped) == expected_keys - ({alias} if alias else set()) + assert skipped[f"{key}_properties"] == metadata if usage: - assert skipped[f"{key}_prompt_tokens"] == 0 + assert skipped[f"{key}_prompt_tokens"] == 9 + + +@pytest.mark.parametrize("strict", [True, False]) +@pytest.mark.parametrize("top_level_usage", [True, False]) +def test_post_inference_skip_preserves_measured_usage(strict, top_level_usage): + configured = strategy(strict=strict, top_level_usage=top_level_usage, completed_alias="legacy_metric") + output = { + "llm_output": { + "status": "skipped", + "score": None, + "reason": "not applicable", + "properties": {"custom": "kept"}, + }, + "input_token_count": 9, + "output_token_count": 3, + "total_token_count": 12, + "model_id": "chat-judge-skip-fixture", + } + configured.runner.return_value = output + result = asyncio.run(configured.evaluate({"query": "q"})) + metadata = { + "prompt_tokens": 9, + "completion_tokens": 3, + "total_tokens": 12, + "model": "chat-judge-skip-fixture", + "finish_reason": "", + "sample_input": "", + "sample_output": "", + } + assert result == { + "metric": None, + "metric_score": None, + "metric_passed": None, + "metric_result": "not_applicable", + "metric_reason": "Not applicable: not applicable", + "metric_status": "skipped", + "metric_threshold": 3, + "metric_properties": {"custom": "kept", **metadata}, + **({f"metric_{key}": value for key, value in metadata.items()} if top_level_usage else {}), + } + configured.runner.assert_awaited_once_with(timeout=17, query="q") + + +@pytest.mark.parametrize("top_level_usage", [True, False]) +def test_pre_inference_skip_does_not_call_runner_or_manufacture_usage(top_level_usage): + configured = strategy(top_level_usage=top_level_usage) + result = configured.skipped("no applicable input") + assert result["metric_status"] == "skipped" + assert result["metric"] is result["metric_score"] is result["metric_passed"] is None + assert result["metric_properties"] is None + if top_level_usage: + assert [result[f"metric_{key}"] for key in ("prompt_tokens", "completion_tokens", "total_tokens")] == [0, 0, 0] + assert result["metric_model"] == "" + else: + assert "metric_prompt_tokens" not in result + assert "metric_model" not in result + configured.runner.assert_not_called() def test_runtime_flow_resolution(): From e3b1653e6034b210920ae0de2504ed791193ef5e Mon Sep 17 00:00:00 2001 From: Ahmad Qardahji Date: Mon, 19 Oct 2026 14:31:57 -0700 Subject: [PATCH 24/24] Preserve Decision error accounting across turns and numeric failures --- .../azure-ai-evaluation/CHANGELOG.md | 2 + .../_batch_run/_run_submitter_client.py | 2 +- .../_evaluators/_common/_base_eval.py | 11 +- .../_evaluators/_common/_base_model_eval.py | 55 +++++ .../_evaluators/_common/_decision_strategy.py | 2 +- .../test_decision_batch_persistence.py | 4 +- .../test_decision_conversation_failure.py | 232 ++++++++++++++++++ .../unittests/test_decision_model_support.py | 30 +++ 8 files changed, 334 insertions(+), 4 deletions(-) create mode 100644 sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_conversation_failure.py diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index 788c45ad08c3..b21706906639 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -16,6 +16,8 @@ ### Bugs Fixed +- Preserved known Decision usage when a later split conversation turn fails, without assigning mixed-model usage to one model or treating the failed turn's execution summary as a conversation summary. +- Kept numeric-overflow validation failures on the typed Decision error path so known usage and provenance survive failed-row conversion. - Preserved measured token usage, model metadata, and response properties when the shared Prompty strategy parses a post-inference skipped result. Pre-inference skips still make no model request. - Hardened local Prompty image resolution so only relative files within the Prompty directory are inlined, with opened-file validation to prevent path changes from redirecting reads and without exposing the local directory in fallback logs. diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py index 2a211c59e5e6..9c0b4774d478 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py @@ -61,7 +61,7 @@ def _decision_error_metadata(exception: EvaluationException) -> Dict[str, Any]: for key, accepted in ( ("backend", ("foundry_decision", "system_one")), ("protocol", ("shared_sdk", "system_one")), - ("result_usage_scope", ("reported_attempts_only", "final_response_only")), + ("result_usage_scope", ("reported_attempts_only", "final_response_only", "reported_turns_only")), ): value = native.get(key) if isinstance(value, str) and value in accepted: diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py index df46ac6424bd..e9842f92cbd1 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py @@ -609,6 +609,11 @@ def _extract_tool_names_and_params_from_response(self, response) -> List[Tuple[s return tool_name_param_pairs + def _on_evaluation_error( + self, error: EvaluationException, per_turn_results: List[DoEvalResult[T_EvalValue]] + ) -> None: + """Allow a model-backed evaluator to retain completed-turn metadata on a later failure.""" + async def _real_call(self, **kwargs) -> Union[DoEvalResult[T_EvalValue], AggregateResult[T_EvalValue]]: """The asynchronous call where real end-to-end evaluation logic is performed. @@ -626,7 +631,11 @@ async def _real_call(self, **kwargs) -> Union[DoEvalResult[T_EvalValue], Aggrega per_turn_results = [] # Evaluate all inputs. for eval_input in eval_input_list: - result = await self._do_eval(eval_input) + try: + result = await self._do_eval(eval_input) + except EvaluationException as error: + self._on_evaluation_error(error, per_turn_results) + raise # logic to determine threshold pass/fail # if it wasn't computed in _do_eval try: diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py index 847195da453c..093fad1bd3fa 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_model_eval.py @@ -235,6 +235,61 @@ def _aggregate_results(self, per_turn_results: List[Dict[str, T]]) -> Dict[str, aggregated.pop(metric, None) return aggregated + def _on_evaluation_error(self, error: EvaluationException, per_turn_results: List[Dict[str, T]]) -> None: + """Preserve known Decision turn usage without changing the failure or Prompty behavior. + + Only the request-owning result's usage is accumulated. A failed turn's execution summary cannot + describe all these turns, so it is omitted and the wider usage scope is explicit. + """ + if not self._uses_decision_model or not per_turn_results: + return + fields = ("prompt_tokens", "completion_tokens", "total_tokens") + contributions = [] + for result in per_turn_results: + usage = self._known_turn_usage(result, fields, prefix=f"{self._result_key}_") + if usage: + contributions.append((usage, result.get(f"{self._result_key}_model"))) + if not contributions: + return + properties = dict(error.properties) if isinstance(error.properties, dict) else {} + failing_usage = self._known_turn_usage(properties, fields) + if failing_usage: + contributions.append((failing_usage, properties.get("model"))) + for field in fields: + known = [usage[field] for usage, _ in contributions if field in usage] + if known: + properties[field] = sum(known) + else: + properties.pop(field, None) + models = [model for _, model in contributions] + common_model = models[0] if isinstance(models[0], str) and models[0].strip() else None + if common_model is not None and all(model == common_model for model in models): + properties["model"] = common_model + else: + properties.pop("model", None) + native = properties.get("decision_model") + native = dict(native) if isinstance(native, dict) else {} + native.pop("execution", None) + if "model" in properties: + native["model"] = properties["model"] + else: + native.pop("model", None) + native["usage_available"] = True + native["result_usage_scope"] = "reported_turns_only" + properties["decision_model"] = native + error.properties = properties + + @staticmethod + def _known_turn_usage(values: Dict[str, Any], fields: tuple, *, prefix: str = "") -> Dict[str, int]: + """Keep each known count independently; missing totals are not inferred from partial counts.""" + return { + field: values[prefix + field] + for field in fields + if isinstance(values.get(prefix + field), int) + and not isinstance(values[prefix + field], bool) + and values[prefix + field] >= 0 + } + def _get_binary_result(self, score: float) -> str: """Get the binary result based on the score. diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py index cb5aaee4a7f3..ee9531f68478 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_decision_strategy.py @@ -24,7 +24,7 @@ def _checked_value(validate: Callable[..., T], error_target: ErrorTarget, *args: Any) -> T: try: return validate(*args) - except ValueError as exc: + except (ValueError, OverflowError) as exc: message = f"The decision model returned an invalid response: {exc}" raise EvaluationException( message=message, diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py index 787b8c4b092c..2cc53a977f59 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_batch_persistence.py @@ -380,7 +380,9 @@ class ToolUseQualityEvaluator(OutputQualityEvaluator): } ] - @pytest.mark.parametrize("status", [float("nan"), float("inf"), "bad"]) + @pytest.mark.parametrize( + "status", [float("nan"), float("inf"), "bad", pytest.param(10**309, id="oversized_integer")] + ) def test_semantic_failure_retains_measured_usage_and_hashes(self, batch_client, tmp_path, status): class Runner: deployment = "Decision-1" diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_conversation_failure.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_conversation_failure.py new file mode 100644 index 000000000000..35588ab0f308 --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_conversation_failure.py @@ -0,0 +1,232 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- + +"""Failed split conversations retain known usage through public calls and batch conversion.""" + +import asyncio +from copy import deepcopy +import json +import logging +from unittest.mock import Mock + +import pandas as pd +import pytest +from azure.core.pipeline.transport import AsyncHttpTransport, HttpResponse + +from azure.ai.evaluation._evaluate._batch_run import RunSubmitterClient +from azure.ai.evaluation._evaluate._evaluate import _convert_single_row_to_aoai_format +from azure.ai.evaluation._evaluators._common import ModelEvaluatorBase +from azure.ai.evaluation._evaluators._common import _model_runners +from azure.ai.evaluation._evaluators._common._decision_strategy import BinaryScoring, DecisionEvaluationStrategy +from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException +from azure.ai.evaluation._http_utils import AsyncHttpPipeline + +pytestmark = pytest.mark.unittest + +CONFIG = { + "type": "decision", + "azure_endpoint": "https://example.services.ai.azure.com", + "azure_deployment": "Decision-1", + "api_key": "offline-key", +} +QUESTIONS = {"status": {"type": "noul"}, "task_completion": {"type": "noul"}} +CONVERSATION = { + "messages": [ + {"role": "user", "content": "first"}, + {"role": "assistant", "content": "one"}, + {"role": "user", "content": "second"}, + {"role": "assistant", "content": "two"}, + ] +} + + +class SplitDecisionEvaluator(ModelEvaluatorBase): + _supports_decision_models = True + + def __init__(self): + super().__init__(result_key="task_completion", model_config=CONFIG, timeout=1, max_retries=0) + self.error = None + self.original_properties = None + + def _derive_singleton_inputs(self): + return [["query", "response"]] + + async def _do_eval(self, eval_input): + try: + return await DecisionEvaluationStrategy( + self._decision_model_runner, + result_key="task_completion", + error_target=ErrorTarget.EVALUATE, + questions=QUESTIONS, + scoring=BinaryScoring(0.75), + threshold=1, + timeout=self._LLM_CALL_TIMEOUT, + ).evaluate(eval_input) + except EvaluationException as error: + if not {"prompt_tokens", "completion_tokens", "total_tokens"}.intersection(error.properties or {}): + # A caller's unmeasured-turn metadata must not label prior known usage unavailable. + error.properties.setdefault("decision_model", {}).update( + usage_available=False, model="unmeasured-failure-model" + ) + self.error = error + self.original_properties = deepcopy(error.properties) + raise + + +@pytest.mark.parametrize("batch", [False, True], ids=["public", "batch"]) +@pytest.mark.parametrize( + "failure,failed_turn,mixed", + [ + ("semantic", 1, False), + ("semantic", 1, True), + ("partial", 1, False), + ("timeout", 1, False), + ("semantic", 0, False), + ("timeout", 0, False), + ], +) +def test_failed_split_conversation_preserves_known_usage(monkeypatch, tmp_path, batch, failure, failed_turn, mixed): + calls = [] + + class Response(HttpResponse): + def __init__(self, request, body): + super().__init__(request, None) + self.status_code, self.headers, self._body = 200, {}, body + + def body(self): + return self._body + + class Transport(AsyncHttpTransport): + async def open(self): + pass + + async def close(self): + pass + + async def __aexit__(self, *_): + await self.close() + + async def send(self, request, **kwargs): + failing = len(calls) == failed_turn + calls.append(json.loads(request.body)) + if failing and failure == "timeout": + await asyncio.sleep(5) + body = { + "model": "other-model" if failing and mixed else "same-model", + "usage": {"input_tokens": 34 if failing else 12, "output_tokens": 5 if failing else 3}, + "answers": { + "status": {"type": "noul", "noul": 0.9}, + "task_completion": {"type": "noul", "noul": 2 if failing else 0.9}, + }, + } + if failing and failure == "partial": + body["usage"]["input_tokens"] = True + return Response(request, json.dumps(body).encode()) + + monkeypatch.setattr(_model_runners, "get_async_http_client", lambda **_: AsyncHttpPipeline(transport=Transport())) + evaluator = SplitDecisionEvaluator() + if batch: + client = RunSubmitterClient() + try: + run = client.run( + evaluator, + pd.DataFrame([{"conversation": CONVERSATION}]), + evaluator_name="judge", + column_mapping={"conversation": "${data.conversation}"}, + ) + details = client.get_details(run, all_results=True).to_dict(orient="records")[0] + summary = client.get_run_summary(run) + row = {key.replace("outputs.", "outputs.judge.", 1): value for key, value in details.items()} + converted = _convert_single_row_to_aoai_format( + row, + 0, + None, + None, + 0, + {"judge": {"metrics": ["task_completion"], "type": "azure_ai_evaluator"}}, + {"judge": summary}, + logging.getLogger("test"), + ) + path = tmp_path / "failed-conversation.json" + path.write_text(json.dumps(converted, allow_nan=False), encoding="utf-8") + result = json.loads(path.read_text(encoding="utf-8"))["results"][0] + assert summary["failed_lines"] == 1 and summary["completed_lines"] == 0 + assert result["status"] == "error" and result["score"] is None and result["passed"] is None + assert result["sample"]["error"]["code"] == "FAILED_EXECUTION" + assert result["sample"]["error"]["blame"] == "SystemError" + properties = result["properties"] + finally: + client._thread_pool.shutdown(wait=True) + else: + with pytest.raises(EvaluationException) as raised: + evaluator(conversation=CONVERSATION) + assert raised.value is evaluator.error + assert raised.value.category == ErrorCategory.FAILED_EXECUTION + assert raised.value.blame == ErrorBlame.SYSTEM_ERROR + properties = raised.value.properties + + assert len(calls) == failed_turn + 1 + if failed_turn: + expected = {"prompt_tokens": 46, "completion_tokens": 8, "total_tokens": 54} + if failure == "timeout": + expected = {"prompt_tokens": 12, "completion_tokens": 3, "total_tokens": 15} + elif failure == "partial": + expected = {"prompt_tokens": 12, "completion_tokens": 8, "total_tokens": 15} + assert {field: properties[field] for field in expected} == expected + assert properties["decision_model"]["result_usage_scope"] == "reported_turns_only" + assert properties["decision_model"]["usage_available"] is True + assert "execution" not in properties["decision_model"] + if mixed: + assert "model" not in properties and "model" not in properties["decision_model"] + else: + assert properties["model"] == "same-model" + assert properties["decision_model"]["model"] == "same-model" + if batch: + assert result["sample"]["usage"] == expected + assert result["sample"].get("model") == (None if mixed else "same-model") + else: + assert "result_usage_scope" not in properties["decision_model"] + assert properties["decision_model"]["execution"]["attempt_count"] == 1 + if failure == "semantic": + assert properties["total_tokens"] == 39 + else: + assert not {"prompt_tokens", "completion_tokens", "total_tokens"}.intersection(properties) + if not batch: + assert properties == evaluator.original_properties + + +def test_prompty_split_failure_preserves_existing_exception(monkeypatch): + monkeypatch.setattr(_model_runners.AsyncPrompty, "load", Mock()) + error = EvaluationException( + message="chat failure", + internal_message="chat failure", + target=ErrorTarget.EVALUATE, + category=ErrorCategory.INVALID_VALUE, + blame=ErrorBlame.USER_ERROR, + ) + error.properties = {"model": "chat-model", "total_tokens": 39, "custom": "unchanged"} + original = deepcopy(error.properties) + + class ChatEvaluator(ModelEvaluatorBase): + def _derive_singleton_inputs(self): + return [["query", "response"]] + + async def _do_eval(self, eval_input): + if eval_input["query"] == "second": + raise error + return {"metric": 1, "metric_total_tokens": 15, "metric_model": "chat-model"} + + evaluator = ChatEvaluator( + result_key="metric", + prompty_file="unused.prompty", + model_config={ + "azure_endpoint": "https://example.openai.azure.com", + "azure_deployment": "chat", + "api_key": "offline-key", + }, + ) + with pytest.raises(EvaluationException) as raised: + evaluator(conversation=CONVERSATION) + assert raised.value is error and error.properties == original + assert error.category == ErrorCategory.INVALID_VALUE and error.blame == ErrorBlame.USER_ERROR diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py index d28bd7d6b9fa..4ed45c0796e7 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_decision_model_support.py @@ -843,6 +843,36 @@ def strategy_with(payload, scoring, key="tool_call_accuracy", extra_properties=N class TestSharedDecisionStrategy: + @pytest.mark.parametrize("field", ["status", "noul", "probability", "confidence", "score"]) + def test_oversized_integer_preserves_typed_failure_metadata(self, field): + from azure.ai.evaluation._evaluate._batch_run._run_submitter_client import _decision_error_metadata + + response = completion_payload(0.9) if field == "noul" else level_payload(4) + if field == "status": + response["answers"]["status"]["noul"] = 10**309 + elif field == "noul": + response["answers"]["task_completion"]["noul"] = 10**309 + elif field == "probability": + response["answers"]["tool_call_accuracy"]["probabilities"]["0"] = 10**309 + else: + response["answers"]["tool_call_accuracy"][field] = 10**309 + response = json.loads(json.dumps(response)) + strategy = ( + strategy_with(response, BinaryScoring(0.75), "task_completion") + if field == "noul" + else strategy_with(response, LikertScoring(5)) + ) + with pytest.raises(EvaluationException) as raised: + strategy.parse(response, {}) + assert raised.value.blame == ErrorBlame.SYSTEM_ERROR + assert raised.value.category == ErrorCategory.FAILED_EXECUTION + metadata = _decision_error_metadata(raised.value) + assert metadata["properties"]["total_tokens"] == 123 + assert metadata["properties"]["model"] == "microsoft-decision-1" + assert metadata["properties"]["decision_model"]["deployment"] == "Decision-1" + assert len(metadata["properties"]["decision_model"]["rubric_hash"]) == 64 + json.dumps(metadata, allow_nan=False) + @pytest.mark.parametrize("status", [0.1, 0.9]) def test_parse_reuses_shared_response_without_http_or_usage_suffixes(self, status): strategy = strategy_with(level_payload(4, status=status), LikertScoring(5))