diff --git a/scripts/regen_metric_signatures.py b/scripts/regen_metric_signatures.py index 9c27e69e..dcc6721a 100644 --- a/scripts/regen_metric_signatures.py +++ b/scripts/regen_metric_signatures.py @@ -11,9 +11,81 @@ """ import json +import sys +from importlib.abc import Loader, MetaPathFinder +from importlib.machinery import ModuleSpec from pathlib import Path +from types import ModuleType +from typing import TypedDict -from eva.metrics.signatures import compute_all_metric_signatures + +def _stub_heavy_imports() -> None: + """Install a meta-path finder that stubs heavy packages before any eva imports. + + litellm and pipecat together account for ~4 s of import time, but + regen_metric_signatures only needs class definitions and source hashes — + it never calls into these libraries at runtime. + + Using a MetaPathFinder means every submodule import (no matter how deep) + is automatically intercepted and given a lightweight stub, with no manual + per-submodule registration required. + + The one exception is DeploymentTypedDict, which is used as a base class in + eva.models.config and must be an actual type, not a stub object. + """ + _STUB_PACKAGES = frozenset({"litellm"}) + + class _AutoStub(ModuleType): + """A module stub that satisfies arbitrary attribute and submodule access.""" + + def __getattr__(self, name: str) -> "_AutoStub": + child = _AutoStub(f"{self.__name__}.{name}") + object.__setattr__(self, name, child) + sys.modules[child.__name__] = child + return child + + def __call__(self, *args: object, **kwargs: object) -> "_AutoStub": + return self + + def __iter__(self): # type: ignore[override] + return iter([]) + + # Allow use in type union expressions at module level, e.g. `Router | None` + def __or__(self, other: object) -> object: + return object + + def __ror__(self, other: object) -> object: + return object + + class _StubLoader(Loader): + def create_module(self, spec: ModuleSpec) -> _AutoStub: + return _AutoStub(spec.name) + + def exec_module(self, module: ModuleType) -> None: + pass # _AutoStub handles everything via __getattr__ + + class _StubFinder(MetaPathFinder): + def find_spec(self, fullname: str, path: object, target: object = None) -> ModuleSpec | None: + if fullname.split(".")[0] in _STUB_PACKAGES: + return ModuleSpec(fullname, _StubLoader()) + return None + + sys.meta_path.insert(0, _StubFinder()) + + # DeploymentTypedDict is inherited by ModelDeployment in eva.models.config, + # so it must be a real class. Trigger the import so the stub is registered in + # sys.modules, then replace the attribute with a proper TypedDict. + import litellm.types.router # noqa: PLC0415, F401 + + class DeploymentTypedDict(TypedDict, total=False): + pass + + sys.modules["litellm.types.router"].DeploymentTypedDict = DeploymentTypedDict # type: ignore[attr-defined] + + +_stub_heavy_imports() + +from eva.metrics.signatures import compute_all_metric_signatures # noqa: E402 REPO_ROOT = Path(__file__).resolve().parent.parent FIXTURE_PATH = REPO_ROOT / "tests" / "fixtures" / "metric_signatures.json" diff --git a/src/eva/assistant/agentic/system.py b/src/eva/assistant/agentic/system.py index 7be55472..12575b45 100644 --- a/src/eva/assistant/agentic/system.py +++ b/src/eva/assistant/agentic/system.py @@ -17,6 +17,7 @@ ) from eva.assistant.tools.tool_executor import ToolExecutor from eva.models.agents import AgentConfig +from eva.utils.conversation_checks import LLM_GENERIC_ERROR_MESSAGE as GENERIC_ERROR from eva.utils.error_handler import categorize_error from eva.utils.log_processing import truncate_data_uris from eva.utils.logging import get_logger @@ -27,9 +28,6 @@ # Suppress LiteLLM's Pydantic serialization warnings (harmless internal warnings) warnings.filterwarnings("ignore", category=UserWarning, message=".*Pydantic serializer warnings.*") -# Response messages -GENERIC_ERROR = "I'm sorry, I encountered an error processing your request." - def _clean_tool_name(name: str) -> str: """Strip Harmony special tokens that leak into tool names due to a known vLLM bug. diff --git a/src/eva/metrics/processor.py b/src/eva/metrics/processor.py index aa4a34b9..2a6f83db 100644 --- a/src/eva/metrics/processor.py +++ b/src/eva/metrics/processor.py @@ -5,9 +5,9 @@ from dataclasses import dataclass, field from pathlib import Path -from eva.assistant.agentic.system import GENERIC_ERROR from eva.models.config import PipelineType from eva.models.results import ConversationResult +from eva.utils.conversation_checks import LLM_GENERIC_ERROR_MESSAGE as GENERIC_ERROR from eva.utils.log_processing import ( AnnotationLabel, aggregate_pipecat_logs_by_type,