From 015e1ea023e3b018c08a41546e104c6cff5ab25c Mon Sep 17 00:00:00 2001 From: Alkis Polyzotis Date: Mon, 26 Jan 2026 19:01:37 +0000 Subject: [PATCH] Merge best documentation from PR #2 into agent-evaluation skill Add new reference documentation: - GOTCHAS.md: 15+ common mistakes that cause evaluation failures - CRITICAL-interfaces.md: Implementation details not in official docs - patterns-context-optimization.md: Token/latency optimization strategies - user-journeys.md: High-level workflow guides Enhance existing references with advanced patterns: - dataset-preparation.md: Production traces, diverse sampling, per-row guidelines - scorers.md: Multi-metric, trace-based, tool selection, latency scorers - tracing-integration.md: Trace analysis patterns for debugging Update SKILL.md with references to new documentation. Co-Authored-By: Claude Opus 4.5 --- agent-evaluation/SKILL.md | 13 +- .../references/CRITICAL-interfaces.md | 319 ++++++++++ agent-evaluation/references/GOTCHAS.md | 547 ++++++++++++++++++ .../references/dataset-preparation.md | 169 ++++++ .../patterns-context-optimization.md | 317 ++++++++++ agent-evaluation/references/scorers.md | 252 ++++++++ .../references/tracing-integration.md | 162 ++++++ agent-evaluation/references/user-journeys.md | 285 +++++++++ 8 files changed, 2061 insertions(+), 3 deletions(-) create mode 100644 agent-evaluation/references/CRITICAL-interfaces.md create mode 100644 agent-evaluation/references/GOTCHAS.md create mode 100644 agent-evaluation/references/patterns-context-optimization.md create mode 100644 agent-evaluation/references/user-journeys.md diff --git a/agent-evaluation/SKILL.md b/agent-evaluation/SKILL.md index e007d5f..763f588 100644 --- a/agent-evaluation/SKILL.md +++ b/agent-evaluation/SKILL.md @@ -255,11 +255,18 @@ uv run python scripts/validate_auth.py # Test authentication before expe Detailed guides in `references/` (load as needed): +### Core Guides - **setup-guide.md** - Environment setup (MLflow install, tracking URI configuration) -- **tracing-integration.md** - Authoritative tracing guide (autolog, decorators, session tracking, verification) -- **dataset-preparation.md** - Dataset schema, APIs, creation, Unity Catalog -- **scorers.md** - Built-in vs custom scorers, registration, testing +- **tracing-integration.md** - Authoritative tracing guide (autolog, decorators, session tracking, verification, trace analysis) +- **dataset-preparation.md** - Dataset schema, APIs, creation, Unity Catalog, advanced patterns +- **scorers.md** - Built-in vs custom scorers, registration, testing, advanced patterns - **scorers-constraints.md** - CLI requirements for custom scorers (yes/no format, templates) - **troubleshooting.md** - Common errors by phase with solutions +### Reference Documentation +- **GOTCHAS.md** - 15+ common mistakes that cause failures (read before writing code) +- **CRITICAL-interfaces.md** - Implementation details not in official docs (data schema, return types, filter syntax) +- **patterns-context-optimization.md** - Token/latency optimization strategies for agents +- **user-journeys.md** - High-level workflow guides (strategy alignment, regression detection, performance) + Scripts are self-documenting - run with `--help` for usage details. diff --git a/agent-evaluation/references/CRITICAL-interfaces.md b/agent-evaluation/references/CRITICAL-interfaces.md new file mode 100644 index 0000000..853029c --- /dev/null +++ b/agent-evaluation/references/CRITICAL-interfaces.md @@ -0,0 +1,319 @@ +# Critical MLflow 3 GenAI Interfaces + +**Purpose**: Implementation details NOT available in MLflow official documentation. +**For general API reference**: Use documentation protocol (llms.txt) +**For common mistakes**: See GOTCHAS.md + +--- + +## Table of Contents + +- [Data Schema](#data-schema) +- [Custom Scorer Return Types](#custom-scorer-return-types) +- [Judges API (Low-level)](#judges-api-low-level) +- [Trace Search Filter Syntax](#trace-search-filter-syntax) +- [Trace Object Access](#trace-object-access) +- [SpanType Constants](#spantype-constants) +- [Production Monitoring API](#production-monitoring-api) + +--- + +## Data Schema + +### Evaluation Dataset Record + +```python +# CORRECT format +record = { + "inputs": { # REQUIRED - passed to predict_fn + "customer_name": "Acme", + "query": "What is X?" + }, + "outputs": { # OPTIONAL - pre-computed outputs + "response": "X is..." + }, + "expectations": { # OPTIONAL - ground truth for scorers + "expected_facts": ["fact1", "fact2"], + "expected_response": "X is...", + "guidelines": ["Must be concise"] + } +} +``` + +**CRITICAL Schema Rules**: +- `inputs` is REQUIRED - contains what's passed to your app +- `outputs` is OPTIONAL - if provided, predict_fn is skipped +- `expectations` is OPTIONAL - used by Correctness, ExpectationsGuidelines + +--- + +## Custom Scorer Return Types + +### Function-based Scorer (Decorator) + +```python +from mlflow.genai.scorers import scorer +from mlflow.entities import Feedback + +@scorer +def my_scorer( + inputs: dict, # From data record + outputs: dict, # App outputs or pre-computed + expectations: dict, # From data record (optional) + trace: Trace = None # Full MLflow Trace object (optional) +) -> Feedback | bool | int | float | str | list[Feedback]: + """Custom scorer implementation""" + + # Return options: + # 1. Simple value (metric name = function name) + return True + + # 2. Feedback object with custom name + return Feedback( + name="custom_metric", + value="yes", # or "no", True/False, int, float + rationale="Explanation of score" + ) + + # 3. Multiple feedbacks + return [ + Feedback(name="metric_1", value=True), + Feedback(name="metric_2", value=0.85) + ] +``` + +### Class-based Scorer + +```python +from mlflow.genai.scorers import Scorer +from mlflow.entities import Feedback +from typing import Optional + +class MyScorer(Scorer): + name: str = "my_scorer" # REQUIRED + threshold: int = 50 # Custom fields allowed (Pydantic) + + def __call__( + self, + outputs: str, + inputs: dict = None, + expectations: dict = None, + trace = None + ) -> Feedback: + if len(outputs) > self.threshold: + return Feedback(value=True, rationale="Meets length requirement") + return Feedback(value=False, rationale="Too short") + +# Usage +my_scorer = MyScorer(threshold=100) +``` + +--- + +## Judges API (Low-level) + +### Import Path +```python +from mlflow.genai.judges import ( + meets_guidelines, + is_correct, + is_safe, + is_context_relevant, + is_grounded, + make_judge, +) +``` + +### meets_guidelines() +```python +from mlflow.genai.judges import meets_guidelines + +feedback = meets_guidelines( + name="my_check", # Optional display name + guidelines="Must be professional", # str or List[str] + context={ # Dict with data to evaluate + "request": "user question", + "response": "app response", + "retrieved_documents": [...] # Can include any keys + }, + model="databricks:/endpoint" # Optional custom model +) +# Returns: Feedback(value="yes"|"no", rationale="...") +``` + +### is_correct() +```python +from mlflow.genai.judges import is_correct + +feedback = is_correct( + request="What is MLflow?", + response="MLflow is an open-source platform...", + expected_facts=["MLflow is open-source"], # OR expected_response + model="databricks:/endpoint" # Optional +) +``` + +### make_judge() - Custom LLM Judge +```python +from mlflow.genai.judges import make_judge + +issue_judge = make_judge( + name="issue_resolution", + instructions=""" + Evaluate if the customer's issue was resolved. + User's messages: {{ inputs }} + Agent's responses: {{ outputs }} + + Rate and respond with exactly one of: + - 'fully_resolved' + - 'partially_resolved' + - 'needs_follow_up' + """, + model="databricks:/databricks-gpt-5-mini" # Optional +) + +# Use in evaluation +results = mlflow.genai.evaluate( + data=eval_dataset, + predict_fn=my_app, + scorers=[issue_judge] +) +``` + +### Trace-based Judge (with {{ trace }}) +```python +# Including {{ trace }} in instructions enables trace exploration +tool_judge = make_judge( + name="tool_correctness", + instructions=""" + Analyze the execution {{ trace }} to determine if appropriate tools were called. + Respond with true or false. + """, + model="databricks:/databricks-gpt-5-mini" # REQUIRED for trace judges +) +``` + +--- + +## Trace Search Filter Syntax + +### Common Filters +```python +import mlflow + +traces_df = mlflow.search_traces( + filter_string="attributes.status = 'OK'", + order_by=["attributes.timestamp_ms DESC"], + max_results=100, + run_id="optional-run-id" # Filter to specific evaluation run +) + +# Common filters: +# "attributes.status = 'OK'" or "attributes.status = 'ERROR'" +# "attributes.timestamp_ms > {milliseconds}" +# "attributes.execution_time_ms > 5000" +# "tags.environment = 'production'" +# "tags.`mlflow.traceName` = 'my_function'" +``` + +### Filter Syntax Rules + +| Syntax Element | Rule | +|----------------|------| +| String values | Use single quotes: `'OK'` NOT `"OK"` | +| Dotted names | Use backticks: `tags.\`mlflow.traceName\`` | +| Prefix | Required for attributes: `attributes.status` | +| Logical operators | `AND` supported, `OR` NOT supported | +| Time values | Use milliseconds since epoch | + +--- + +## Trace Object Access + +```python +from mlflow.entities import Trace, SpanType + +@scorer +def trace_scorer(trace: Trace) -> Feedback: + # Search spans by type + llm_spans = trace.search_spans(span_type=SpanType.CHAT_MODEL) + retriever_spans = trace.search_spans(span_type=SpanType.RETRIEVER) + + # Access span data + for span in llm_spans: + duration = (span.end_time_ns - span.start_time_ns) / 1e9 + inputs = span.inputs + outputs = span.outputs +``` + +--- + +## SpanType Constants + +```python +from mlflow.entities import SpanType + +SpanType.CHAT_MODEL # LLM calls +SpanType.RETRIEVER # RAG retrieval +SpanType.TOOL # Tool/function calls +SpanType.AGENT # Agent execution +SpanType.CHAIN # Chain execution +``` + +### Feedback Values +```python +# LLM judges typically return: +"yes" | "no" # For pass/fail assessments + +# Custom scorers can return: +True | False # Boolean +0.0 - 1.0 # Float scores +int # Integer scores +str # Categorical values +``` + +--- + +## Production Monitoring API + +### Register and Start Scorer +```python +from mlflow.genai.scorers import Safety, Guidelines, ScorerSamplingConfig + +# Register scorer to experiment +safety = Safety().register(name="safety_monitor") + +# Start monitoring with sample rate +safety = safety.start( + sampling_config=ScorerSamplingConfig(sample_rate=0.5) # 50% of traces +) +``` + +### ScorerSamplingConfig Options +```python +ScorerSamplingConfig( + sample_rate=0.5, # Sample 50% of traces (0.0 to 1.0) +) +``` + +### Manage Scorers +```python +from mlflow.genai.scorers import list_scorers, get_scorer, delete_scorer + +# List all registered scorers +scorers = list_scorers() + +# Get specific scorer +my_scorer = get_scorer(name="safety_monitor") + +# Update sample rate +my_scorer = my_scorer.update( + sampling_config=ScorerSamplingConfig(sample_rate=0.8) +) + +# Stop monitoring (keeps registration) +my_scorer = my_scorer.stop() + +# Delete entirely +delete_scorer(name="safety_monitor") +``` diff --git a/agent-evaluation/references/GOTCHAS.md b/agent-evaluation/references/GOTCHAS.md new file mode 100644 index 0000000..e34f246 --- /dev/null +++ b/agent-evaluation/references/GOTCHAS.md @@ -0,0 +1,547 @@ +# MLflow 3 GenAI - GOTCHAS & Common Mistakes + +**CRITICAL**: Read this before writing any evaluation code. These are the most common mistakes that will cause failures. + +## Table of Contents + +- [Using Model Serving Endpoints for Development](#-wrong-using-model-serving-endpoints-for-development) +- [Wrong API Imports](#-wrong-api-imports) +- [Wrong Evaluate Function](#-wrong-evaluate-function) +- [Wrong Data Format](#-wrong-data-format) +- [Wrong predict_fn Signature](#-wrong-predict_fn-signature) +- [Wrong Scorer Decorator Usage](#-wrong-scorer-decorator-usage) +- [Wrong Feedback Return](#-wrong-feedback-return) +- [Wrong Guidelines Scorer Setup](#-wrong-guidelines-scorer-setup) +- [Wrong Trace Search Syntax](#-wrong-trace-search-syntax) +- [Wrong Expectations Usage](#-wrong-expectations-usage) +- [Wrong RetrievalGroundedness Usage](#-wrong-retrievalgroundedness-usage) +- [Wrong Custom Scorer Imports](#-wrong-custom-scorer-imports) +- [Wrong Type Hints in Scorers](#-wrong-type-hints-in-scorers) +- [Wrong Dataset Creation](#-wrong-dataset-creation) +- [Wrong Multiple Feedback Names](#-wrong-multiple-feedback-names) +- [Wrong Guidelines Context Reference](#-wrong-guidelines-context-reference) +- [Wrong Production Monitoring Setup](#-wrong-production-monitoring-setup) +- [Wrong Custom Judge Model Format](#-wrong-custom-judge-model-format) +- [Wrong Aggregation Values](#-wrong-aggregation-values) +- [Summary Checklist](#summary-checklist) + +--- + +## Wrong: Using Model Serving Endpoints for Development + +### WRONG: Calling deployed endpoint for initial testing +```python +# WRONG - Don't use model serving endpoints during development +from databricks.sdk import WorkspaceClient + +w = WorkspaceClient() +client = w.serving_endpoints.get_open_ai_client() + +def predict_fn(messages): + response = client.chat.completions.create( + model="my-agent-endpoint", # Deployed endpoint + messages=messages + ) + return {"response": response.choices[0].message.content} +``` + +### CORRECT: Import and test agent locally +```python +# CORRECT - Import agent directly for fast iteration +from plan_execute_agent import AGENT # Your local agent module + +def predict_fn(messages): + result = AGENT.predict({"messages": messages}) + # Extract response from ResponsesAgent format + if isinstance(result, dict) and "messages" in result: + for msg in reversed(result["messages"]): + if msg.get("role") == "assistant": + return {"response": msg.get("content", "")} + return {"response": str(result)} +``` + +**Why?** +- Local testing enables faster iteration (no deployment needed) +- Full stack traces for debugging +- No serving endpoint costs +- Direct access to agent internals + +**When to use endpoints**: Only for production monitoring, load testing, or A/B testing deployed versions. + +--- + +## Wrong API IMPORTS + +### WRONG: Using old MLflow 2 imports +```python +# WRONG - These don't exist in MLflow 3 GenAI +from mlflow.evaluate import evaluate +from mlflow.metrics import genai +import mlflow.llm +``` + +### CORRECT: MLflow 3 GenAI imports +```python +# CORRECT +import mlflow.genai +from mlflow.genai.scorers import Guidelines, Safety, Correctness, scorer +from mlflow.genai.judges import meets_guidelines, is_correct, make_judge +from mlflow.entities import Feedback, Trace +``` + +--- + +## Wrong EVALUATE FUNCTION + +### WRONG: Using mlflow.evaluate() +```python +# WRONG - This is the old API for classic ML +results = mlflow.evaluate( + model=my_model, + data=eval_data, + model_type="text" +) +``` + +### CORRECT: Using mlflow.genai.evaluate() +```python +# CORRECT - MLflow 3 GenAI evaluation +results = mlflow.genai.evaluate( + data=eval_dataset, + predict_fn=my_app, + scorers=[Guidelines(name="test", guidelines="...")] +) +``` + +--- + +## Wrong DATA FORMAT + +### WRONG: Flat data structure +```python +# WRONG - Missing nested structure +eval_data = [ + {"query": "What is X?", "expected": "X is..."} +] +``` + +### CORRECT: Proper nested structure +```python +# CORRECT - Must have 'inputs' key +eval_data = [ + { + "inputs": {"query": "What is X?"}, + "expectations": {"expected_response": "X is..."} + } +] +``` + +--- + +## Wrong predict_fn SIGNATURE + +### WRONG: Function expects dict +```python +# WRONG - predict_fn receives **unpacked inputs +def my_app(inputs): # Receives dict + query = inputs["query"] + return {"response": "..."} +``` + +### CORRECT: Function receives keyword args +```python +# CORRECT - inputs are unpacked as kwargs +def my_app(query, context=None): # Receives individual keys + return {"response": f"Answer to {query}"} + +# If inputs = {"query": "What is X?", "context": "..."} +# Then my_app is called as: my_app(query="What is X?", context="...") +``` + +--- + +## Wrong SCORER DECORATOR USAGE + +### WRONG: Missing decorator +```python +# WRONG - This won't work as a scorer +def my_scorer(inputs, outputs): + return True +``` + +### CORRECT: Use @scorer decorator +```python +# CORRECT +from mlflow.genai.scorers import scorer + +@scorer +def my_scorer(inputs, outputs): + return True +``` + +--- + +## Wrong FEEDBACK RETURN + +### WRONG: Returning wrong types +```python +@scorer +def bad_scorer(outputs): + # WRONG - Can't return dict + return {"score": 0.5, "reason": "..."} + + # WRONG - Can't return tuple + return (True, "rationale") +``` + +### CORRECT: Return Feedback or primitive +```python +from mlflow.entities import Feedback + +@scorer +def good_scorer(outputs): + # CORRECT - Return primitive + return True + return 0.85 + return "yes" + + # CORRECT - Return Feedback object + return Feedback( + value=True, + rationale="Explanation" + ) + + # CORRECT - Return list of Feedbacks + return [ + Feedback(name="metric_1", value=True), + Feedback(name="metric_2", value=0.9) + ] +``` + +--- + +## Wrong GUIDELINES SCORER SETUP + +### WRONG: Missing required parameters +```python +# WRONG - Missing 'name' parameter +scorer = Guidelines(guidelines="Must be professional") +``` + +### CORRECT: Include name and guidelines +```python +# CORRECT +scorer = Guidelines( + name="professional_tone", # REQUIRED + guidelines="The response must be professional" # REQUIRED +) +``` + +--- + +## Wrong TRACE SEARCH SYNTAX + +### WRONG: Missing prefixes and wrong quotes +```python +# WRONG - Missing prefix +mlflow.search_traces("status = 'OK'") + +# WRONG - Using double quotes +mlflow.search_traces('attributes.status = "OK"') + +# WRONG - Missing backticks for dotted names +mlflow.search_traces("tags.mlflow.traceName = 'my_app'") + +# WRONG - Using OR (not supported) +mlflow.search_traces("attributes.status = 'OK' OR attributes.status = 'ERROR'") +``` + +### CORRECT: Proper filter syntax +```python +# CORRECT - Use prefix and single quotes +mlflow.search_traces("attributes.status = 'OK'") + +# CORRECT - Backticks for dotted names +mlflow.search_traces("tags.`mlflow.traceName` = 'my_app'") + +# CORRECT - AND is supported +mlflow.search_traces("attributes.status = 'OK' AND tags.env = 'prod'") + +# CORRECT - Time in milliseconds +import time +cutoff = int((time.time() - 3600) * 1000) # 1 hour ago +mlflow.search_traces(f"attributes.timestamp_ms > {cutoff}") +``` + +--- + +## Wrong EXPECTATIONS USAGE + +### WRONG: Using Correctness without expectations +```python +# WRONG - Correctness requires expected_facts or expected_response +eval_data = [ + {"inputs": {"query": "What is X?"}} +] +results = mlflow.genai.evaluate( + data=eval_data, + predict_fn=my_app, + scorers=[Correctness()] # Will fail - no ground truth! +) +``` + +### CORRECT: Include expectations for Correctness +```python +# CORRECT +eval_data = [ + { + "inputs": {"query": "What is X?"}, + "expectations": { + "expected_facts": ["X is a platform", "X is open-source"] + } + } +] +``` + +--- + +## Wrong RetrievalGroundedness USAGE + +### WRONG: Using without RETRIEVER span +```python +# WRONG - App has no RETRIEVER span type +@mlflow.trace +def my_rag_app(query): + docs = get_documents(query) # Not marked as retriever + return generate_response(docs, query) + +# RetrievalGroundedness will fail - can't find retriever spans +``` + +### CORRECT: Mark retrieval with proper span type +```python +# CORRECT - Use span_type="RETRIEVER" +@mlflow.trace(span_type="RETRIEVER") +def retrieve_documents(query): + return [doc1, doc2] + +@mlflow.trace +def my_rag_app(query): + docs = retrieve_documents(query) # Now has RETRIEVER span + return generate_response(docs, query) +``` + +--- + +## Wrong CUSTOM SCORER IMPORTS + +### WRONG: External imports at module level +```python +# WRONG for production monitoring - external import outside function +import my_custom_library + +@scorer +def production_scorer(outputs): + return my_custom_library.process(outputs) +``` + +### CORRECT: Inline imports for production scorers +```python +# CORRECT - Import inside function for serialization +@scorer +def production_scorer(outputs): + import json # Import inside for production monitoring + return len(json.dumps(outputs)) > 100 +``` + +--- + +## Wrong TYPE HINTS IN SCORERS + +### WRONG: Type hints requiring imports in signature +```python +# WRONG - Type hints break serialization for production monitoring +from typing import List + +@scorer +def bad_scorer(outputs: List[str]) -> bool: + return True +``` + +### CORRECT: Avoid complex type hints or use dict +```python +# CORRECT - Simple types work +@scorer +def good_scorer(outputs): + return True + +# CORRECT - dict is fine +@scorer +def good_scorer(outputs: dict) -> bool: + return True +``` + +--- + +## Wrong Dataset Creation + +### WRONG: Missing Spark session for MLflow datasets +```python +# WRONG - Need Spark for MLflow-managed datasets +import mlflow.genai.datasets + +dataset = mlflow.genai.datasets.create_dataset( + uc_table_name="catalog.schema.my_dataset" +) +# Error: No Spark session available +``` + +### CORRECT: Initialize Spark first +```python +# CORRECT +from databricks.connect import DatabricksSession + +spark = DatabricksSession.builder.remote(serverless=True).getOrCreate() + +dataset = mlflow.genai.datasets.create_dataset( + uc_table_name="catalog.schema.my_dataset" +) +``` + +--- + +## Wrong Multiple Feedback Names + +### WRONG: Multiple feedbacks without unique names +```python +@scorer +def bad_multi_scorer(outputs): + # WRONG - Feedbacks will conflict + return [ + Feedback(value=True), + Feedback(value=0.8) + ] +``` + +### CORRECT: Unique names for each Feedback +```python +@scorer +def good_multi_scorer(outputs): + # CORRECT - Each has unique name + return [ + Feedback(name="check_1", value=True), + Feedback(name="check_2", value=0.8) + ] +``` + +--- + +## Wrong Guidelines Context Reference + +### WRONG: Wrong variable names in guidelines +```python +# WRONG - Guidelines use 'request' and 'response', not custom keys +Guidelines( + name="check", + guidelines="The output must address the query" # 'output' and 'query' not available +) +``` + +### CORRECT: Use 'request' and 'response' +```python +# CORRECT - These are auto-extracted +Guidelines( + name="check", + guidelines="The response must address the request" +) +``` + +--- + +## Wrong Production Monitoring Setup + +### WRONG: Forgetting to start after register +```python +# WRONG - Registered but not started +from mlflow.genai.scorers import Safety + +safety = Safety().register(name="safety_check") +# Scorer exists but isn't running! +``` + +### CORRECT: Register then start +```python +# CORRECT - Both register and start +from mlflow.genai.scorers import Safety, ScorerSamplingConfig + +safety = Safety().register(name="safety_check") +safety = safety.start( + sampling_config=ScorerSamplingConfig(sample_rate=0.5) +) +``` + +--- + +## Wrong Custom Judge Model Format + +### WRONG: Wrong model format +```python +# WRONG - Missing provider prefix +Guidelines(name="test", guidelines="...", model="gpt-4o") + +# WRONG - Wrong separator +Guidelines(name="test", guidelines="...", model="databricks:gpt-4o") +``` + +### CORRECT: Use provider:/model format +```python +# CORRECT - Use :/ separator +Guidelines(name="test", guidelines="...", model="databricks:/my-endpoint") +Guidelines(name="test", guidelines="...", model="openai:/gpt-4o") +``` + +--- + +## Wrong Aggregation Values + +### WRONG: Invalid aggregation names +```python +# WRONG - p50, p99, sum are not valid +@scorer(aggregations=["mean", "p50", "p99", "sum"]) +def my_scorer(outputs) -> float: + return 0.5 +``` + +### CORRECT: Use valid aggregation names +```python +# CORRECT - Only these 6 are valid +@scorer(aggregations=["min", "max", "mean", "median", "variance", "p90"]) +def my_scorer(outputs) -> float: + return 0.5 +``` + +**Valid aggregations:** +- `min` - minimum value +- `max` - maximum value +- `mean` - average value +- `median` - 50th percentile (NOT `p50`) +- `variance` - statistical variance +- `p90` - 90th percentile (only p90, NOT p50 or p99) + +--- + +## Summary Checklist + +Before running evaluation, verify: + +- [ ] Using `mlflow.genai.evaluate()` (not `mlflow.evaluate()`) +- [ ] Data has `inputs` key (nested structure) +- [ ] `predict_fn` accepts **unpacked kwargs (not dict) +- [ ] Scorers have `@scorer` decorator +- [ ] Guidelines have both `name` and `guidelines` +- [ ] Correctness has `expectations.expected_facts` or `expected_response` +- [ ] RetrievalGroundedness has `RETRIEVER` span in trace +- [ ] Trace filters use `attributes.` prefix and single quotes +- [ ] Production scorers have inline imports +- [ ] Multiple Feedbacks have unique names +- [ ] Aggregations use valid names: min, max, mean, median, variance, p90 diff --git a/agent-evaluation/references/dataset-preparation.md b/agent-evaluation/references/dataset-preparation.md index 2830c3c..15ebb9f 100644 --- a/agent-evaluation/references/dataset-preparation.md +++ b/agent-evaluation/references/dataset-preparation.md @@ -349,3 +349,172 @@ More queries = better coverage but longer evaluation time and higher LLM costs. --- **For troubleshooting dataset creation issues**, see `references/troubleshooting.md` + +--- + +## Advanced Dataset Patterns + +### Pattern: Dataset from Production Traces + +Convert real traffic into evaluation data. + +```python +import mlflow +import time + +# Search recent production traces +one_week_ago = int((time.time() - 7 * 86400) * 1000) + +prod_traces = mlflow.search_traces( + filter_string=f""" + attributes.status = 'OK' AND + attributes.timestamp_ms > {one_week_ago} AND + tags.environment = 'production' + """, + order_by=["attributes.timestamp_ms DESC"], + max_results=100 +) + +# Convert to eval format (without outputs - will re-run) +eval_data = [] +for _, trace in prod_traces.iterrows(): + eval_data.append({ + "inputs": trace['request'] # request is already a dict + }) + +# Or with outputs (evaluate existing responses) +eval_data_with_outputs = [] +for _, trace in prod_traces.iterrows(): + eval_data_with_outputs.append({ + "inputs": trace['request'], + "outputs": trace['response'] + }) +``` + +### Pattern: Build Diverse Evaluation Dataset + +Sample across different characteristics for comprehensive coverage. + +```python +import pandas as pd + +def build_diverse_eval_dataset(traces_df, sample_size=50): + """ + Build a diverse evaluation dataset from traces. + Samples across different characteristics. + """ + samples = [] + + # Sample by status + ok_traces = traces_df[traces_df['status'] == 'OK'] + error_traces = traces_df[traces_df['status'] == 'ERROR'] + + # Sample by latency buckets + fast = ok_traces[ok_traces['execution_time_ms'] < 1000] + medium = ok_traces[(ok_traces['execution_time_ms'] >= 1000) & + (ok_traces['execution_time_ms'] < 5000)] + slow = ok_traces[ok_traces['execution_time_ms'] >= 5000] + + # Proportional sampling + samples_per_bucket = sample_size // 4 + + if len(fast) > 0: + samples.append(fast.sample(min(samples_per_bucket, len(fast)))) + if len(medium) > 0: + samples.append(medium.sample(min(samples_per_bucket, len(medium)))) + if len(slow) > 0: + samples.append(slow.sample(min(samples_per_bucket, len(slow)))) + if len(error_traces) > 0: + samples.append(error_traces.sample(min(samples_per_bucket, len(error_traces)))) + + # Combine and convert to eval format + combined = pd.concat(samples, ignore_index=True) + + eval_data = [] + for _, row in combined.iterrows(): + eval_data.append({ + "inputs": row['request'], + "outputs": row['response'] + }) + + return eval_data +``` + +### Pattern: Dataset with Per-Row Guidelines + +For row-specific evaluation criteria using ExpectationsGuidelines. + +```python +eval_data = [ + { + "inputs": {"query": "Explain quantum computing"}, + "expectations": { + "guidelines": [ + "Must explain in simple terms", + "Must avoid excessive jargon", + "Must include an analogy" + ] + } + }, + { + "inputs": {"query": "Write code to sort a list"}, + "expectations": { + "guidelines": [ + "Must include working code", + "Must include comments", + "Must mention time complexity" + ] + } + } +] + +# Use with ExpectationsGuidelines scorer +from mlflow.genai.scorers import ExpectationsGuidelines + +results = mlflow.genai.evaluate( + data=eval_data, + predict_fn=my_app, + scorers=[ExpectationsGuidelines()] +) +``` + +### Pattern: Dataset with Stage Expectations + +For multi-agent pipelines, include expectations for each stage. + +```python +eval_data = [ + { + "inputs": { + "question": "What are the top 10 GenAI growth accounts for MFG?" + }, + "expectations": { + # Standard MLflow expectations + "expected_facts": ["growth", "accounts", "MFG", "GenAI"], + + # Stage-specific expectations for custom scorers + "expected_query_type": "growth_analysis", + "expected_tools": ["get_genai_consumption_growth"], + "expected_filters": {"vertical": "MFG"} + }, + "metadata": { + "test_id": "test_001", + "category": "growth_analysis", + "difficulty": "easy" + } + } +] +``` + +### Dataset Categories Checklist + +Ensure coverage across these categories: + +| Category | Purpose | Example | +|----------|---------|---------| +| **Happy Path** | Core functionality | Typical user questions | +| **Edge Cases** | Boundary conditions | Empty inputs, very long queries | +| **Adversarial** | Robustness | Prompt injection, off-topic | +| **Out of Scope** | Graceful decline | Questions agent should refuse | +| **Multi-turn** | Conversation handling | Follow-up questions | +| **Error Recovery** | Invalid inputs | Malformed data, null values | diff --git a/agent-evaluation/references/patterns-context-optimization.md b/agent-evaluation/references/patterns-context-optimization.md new file mode 100644 index 0000000..46e9b29 --- /dev/null +++ b/agent-evaluation/references/patterns-context-optimization.md @@ -0,0 +1,317 @@ +# Context Optimization Strategies + +A guide to managing context windows effectively in agentic systems. These strategies apply across architectures and help maintain quality while reducing token usage. + +## Table of Contents + +- [Why Context Optimization Matters](#why-context-optimization-matters) +- [Strategy 1: Tool Result Management](#strategy-1-tool-result-management) +- [Strategy 2: Message History Compression](#strategy-2-message-history-compression) +- [Strategy 3: Structured State vs. Message History](#strategy-3-structured-state-vs-message-history) +- [Strategy 4: Prompt Engineering for Context Efficiency](#strategy-4-prompt-engineering-for-context-efficiency) +- [Strategy 5: Intelligent Caching](#strategy-5-intelligent-caching) +- [Strategy 6: Compression Triggers](#strategy-6-compression-triggers) +- [Strategy 7: Architecture-Specific Patterns](#strategy-7-architecture-specific-patterns) +- [Metrics for Context Optimization](#metrics-for-context-optimization) +- [Common Pitfalls](#common-pitfalls) +- [Implementation Priority](#implementation-priority) + +--- + +## Why Context Optimization Matters + +Context windows are finite and expensive. Poor context management leads to: +- **Token bloat**: Paying for redundant or low-value tokens +- **Lost context**: Important information pushed out by verbose content +- **Quality degradation**: Model attention diluted across irrelevant content +- **Latency**: Larger contexts = slower inference + +--- + +## Strategy 1: Tool Result Management + +Tool calls often return verbose JSON that quickly fills context windows. + +### Problem +A single tool call might return 5,000+ tokens of JSON data, but only 50 tokens are actually needed for the agent's response. + +### Solutions + +**Selective Field Extraction** +- Before returning tool results to the agent, extract only the fields needed +- Define "essential fields" per tool type (e.g., for a search tool: title, snippet, url) +- Discard metadata, debugging info, and redundant fields + +**Result Truncation** +- Limit array results to top N items (e.g., top 10 search results, not 100) +- Truncate long text fields to first N characters +- Summarize large datasets into aggregates (counts, averages, ranges) + +**Structured Summaries** +- Convert raw tool output to natural language summaries +- "Found 47 results. Top 3: [Company A] (45% growth), [Company B] (32% growth), [Company C] (28% growth)" +- Preserves key facts, drops JSON verbosity + +### When to Apply +- Immediately after tool execution, before adding to context +- More aggressive for older tool results, preserve detail for recent ones + +--- + +## Strategy 2: Message History Compression + +Conversation history grows with each turn. Managing it is critical for multi-turn agents. + +### Tier 1: Sliding Window +Keep only the last N messages, drop older ones. + +| Pros | Cons | +|------|------| +| Simple to implement | Loses historical context | +| Predictable context size | May break conversation continuity | +| No additional latency | User references to old content fail | + +**Best for**: Simple chat agents, short conversations, stateless interactions + +### Tier 2: Filter + Summarize +Filter verbose messages, create summaries of older content. + +| Pros | Cons | +|------|------| +| Preserves key information | Requires extraction logic | +| Good compression (50-70%) | Some detail loss | +| Maintains continuity | Added complexity | + +**Best for**: Tool-calling agents, multi-step tasks, medium-length conversations + +### Tier 3: Semantic Compression +Use an LLM to summarize older conversation segments. + +| Pros | Cons | +|------|------| +| Highest compression (70-85%) | Adds latency (LLM call) | +| Preserves meaning well | Costs tokens for summary | +| Handles complex context | May lose fine details | + +**Best for**: Very long conversations, periodic checkpoints, complex multi-agent workflows + +### Hybrid Approach +Combine tiers based on message age: +- **Recent (last 5-10 messages)**: Keep verbatim +- **Medium (10-30 messages back)**: Tier 2 filtering +- **Old (30+ messages)**: Tier 3 semantic summary + +--- + +## Strategy 3: Structured State vs. Message History + +Instead of passing full message history, maintain structured state that captures conversation semantics. + +### Message History Approach +``` +[Message 1: User asks about X] +[Message 2: Assistant responds] +[Message 3: Tool call result - 2000 tokens of JSON] +[Message 4: Assistant analyzes] +[Message 5: User follow-up about Y] +... +``` +Grows linearly, contains redundancy. + +### Structured State Approach +``` +{ + "topic": "X analysis", + "entities_discussed": ["Company A", "Company B"], + "filters_applied": {"time_range": "Q3 2024"}, + "key_findings": ["Finding 1", "Finding 2"], + "last_query": "Y follow-up" +} +``` +Fixed size, captures semantics. + +### Trade-offs + +| Aspect | Message History | Structured State | +|--------|-----------------|------------------| +| Size growth | Linear | Bounded | +| Context richness | High | Medium | +| Implementation | Simple | Complex | +| Error recovery | Easy (replay) | Harder | +| Multi-turn coherence | Natural | Requires design | + +### Recommendation +Use structured state for: +- Long-running conversations (10+ turns) +- Multi-agent systems (state passed between agents) +- Streaming contexts (state can be serialized/resumed) + +Use message history for: +- Short interactions (< 10 turns) +- Simple Q&A agents +- When full conversation context is genuinely needed + +--- + +## Strategy 4: Prompt Engineering for Context Efficiency + +The system prompt itself can bloat context. Optimize it. + +### Avoid Redundancy +- Don't repeat instructions that are implicit in examples +- Don't include examples that cover the same case +- Reference external docs rather than inlining them + +### Use Hierarchical Instructions +``` +## Core Rules (always apply) +- Rule 1 +- Rule 2 + +## Situational Rules (apply when relevant) +- If X, then Y +- If A, then B +``` +Agent can skip irrelevant sections mentally. + +### Dynamic Prompt Assembly +Instead of a monolithic system prompt, assemble based on context: +- Base instructions (always included) +- Tool-specific guidance (only when tools are bound) +- Domain context (only when relevant to query) + +### Measure Prompt Token Cost +Track tokens used by: +- System prompt (fixed cost per request) +- Few-shot examples (fixed cost) +- Conversation history (variable) +- Tool results (variable, often largest) + +--- + +## Strategy 5: Intelligent Caching + +Avoid redundant computation and token usage through caching. + +### Result Caching +- Cache tool results for identical queries +- Set TTL based on data freshness requirements +- Invalidate on relevant state changes + +### Summary Caching +- Cache computed summaries of conversation segments +- Reuse when that segment hasn't changed +- Particularly valuable for Tier 3 semantic summaries + +### Prompt Caching (Model-Level) +Some providers cache prompt prefixes: +- Anthropic: Automatic prefix caching for repeated prompts +- OpenAI: Prompt caching for identical prefix sequences + +Structure prompts to maximize cache hits: +- Put stable content (system prompt, examples) first +- Put variable content (conversation, tool results) last + +--- + +## Strategy 6: Compression Triggers + +Don't compress on every turn--compress when needed. + +### Signal-Based Triggers +- **Token count**: Compress when estimated tokens > threshold +- **Message count**: Compress when messages > threshold +- **Turn count**: Compress every N turns +- **Time-based**: Compress after N minutes of conversation + +### Threshold Guidelines + +| Agent Type | Token Trigger | Message Trigger | +|------------|---------------|-----------------| +| Simple Chat | 80K | 30 messages | +| RAG Agent | 40K | 15 messages | +| Tool-Calling | 50K | 20 messages | +| Multi-Agent | 30K | 10 messages | + +### Avoid Over-Compression +- Don't compress before you have meaningful content to compress +- Keep recent context intact (last 5-10 messages) +- Verify compression doesn't break agent behavior (test with evals) + +--- + +## Strategy 7: Architecture-Specific Patterns + +### For Multi-Agent Pipelines +- Each agent should receive only the context it needs +- Pass structured summaries between stages, not full history +- The final "executor" stage may need more context than the "classifier" + +### For RAG Agents +- Retrieved documents often dominate context +- Limit chunks returned (top 3-5, not 10+) +- Summarize retrieved content before adding to context +- Consider relevance filtering before retrieval + +### For Streaming Agents +- Context must be serializable for resume +- Prefer structured state over message history +- Compress before serialization checkpoints + +--- + +## Metrics for Context Optimization + +Track these to measure optimization effectiveness: + +| Metric | What It Measures | Target | +|--------|------------------|--------| +| Tokens per request | Context efficiency | Minimize | +| Compression ratio | Before/after tokens | 0.3-0.7 | +| Eval score post-compression | Quality maintenance | No regression | +| Latency impact | Compression overhead | < 100ms | +| Cache hit rate | Redundant computation avoided | > 50% | + +--- + +## Common Pitfalls + +### 1. Compressing Too Aggressively +**Symptom**: Agent can't answer follow-up questions +**Fix**: Preserve recent messages, test with multi-turn evals + +### 2. Ignoring Tool Result Size +**Symptom**: Single tool call fills context window +**Fix**: Truncate/summarize tool results immediately + +### 3. Redundant Context Across Agents +**Symptom**: Multi-agent system passes same content to every stage +**Fix**: Tailor context per agent role + +### 4. No Compression Testing +**Symptom**: Compression breaks edge cases +**Fix**: Include compression scenarios in evaluation dataset + +### 5. Static Thresholds +**Symptom**: Works for some queries, fails for others +**Fix**: Use multi-signal triggers (tokens AND messages AND time) + +--- + +## Implementation Priority + +When implementing context optimization: + +1. **Start with tool results** - Often the biggest win with lowest effort +2. **Add sliding window** - Simple message limit prevents runaway growth +3. **Implement structured extraction** - Capture key facts before filtering +4. **Add compression triggers** - Compress only when needed +5. **Consider semantic summarization** - For complex, long-running conversations + +--- + +## References + +- Anthropic prompt caching: Automatic prefix caching for repeated prompts +- Token estimation: ~4 characters per token heuristic for English text +- Context window limits vary by model--check provider documentation diff --git a/agent-evaluation/references/scorers.md b/agent-evaluation/references/scorers.md index 6557043..28af121 100644 --- a/agent-evaluation/references/scorers.md +++ b/agent-evaluation/references/scorers.md @@ -428,4 +428,256 @@ Review each output to verify scorer behaves as expected. --- +## Advanced Scorer Patterns + +### Pattern: Custom Scorer with Multiple Metrics + +Return multiple metrics from a single scorer. + +```python +from mlflow.genai.scorers import scorer +from mlflow.entities import Feedback + +@scorer +def comprehensive_check(inputs, outputs): + """Return multiple metrics from one scorer.""" + response = str(outputs.get("response", "")) + query = inputs.get("query", "") + + feedbacks = [] + + # Check 1: Response exists + feedbacks.append(Feedback( + name="has_response", + value=len(response) > 0, + rationale="Response is present" if response else "No response" + )) + + # Check 2: Word count + word_count = len(response.split()) + feedbacks.append(Feedback( + name="word_count", + value=word_count, + rationale=f"Response contains {word_count} words" + )) + + # Check 3: Query terms in response + query_terms = set(query.lower().split()) + response_terms = set(response.lower().split()) + overlap = len(query_terms & response_terms) / len(query_terms) if query_terms else 0 + feedbacks.append(Feedback( + name="query_coverage", + value=round(overlap, 2), + rationale=f"{overlap*100:.0f}% of query terms found in response" + )) + + return feedbacks +``` + +### Pattern: Trace-Based Scorer + +Analyze execution details from the trace. + +```python +from mlflow.genai.scorers import scorer +from mlflow.entities import Feedback, Trace, SpanType + +@scorer +def llm_latency_check(trace: Trace) -> Feedback: + """Check if LLM response time is acceptable.""" + + # Find LLM spans in trace + llm_spans = trace.search_spans(span_type=SpanType.CHAT_MODEL) + + if not llm_spans: + return Feedback( + value="no", + rationale="No LLM calls found in trace" + ) + + # Calculate total LLM time + total_llm_time = 0 + for span in llm_spans: + duration = (span.end_time_ns - span.start_time_ns) / 1e9 + total_llm_time += duration + + max_acceptable = 5.0 # seconds + + if total_llm_time <= max_acceptable: + return Feedback( + value="yes", + rationale=f"LLM latency {total_llm_time:.2f}s within {max_acceptable}s limit" + ) + else: + return Feedback( + value="no", + rationale=f"LLM latency {total_llm_time:.2f}s exceeds {max_acceptable}s limit" + ) +``` + +### Pattern: Tool Selection Accuracy Scorer + +Check if the correct tools were called during agent execution. + +```python +from mlflow.genai.scorers import scorer +from mlflow.entities import Feedback, Trace, SpanType +from typing import Dict, Any + +@scorer +def tool_selection_accuracy( + inputs: Dict[str, Any], + outputs: Dict[str, Any], + expectations: Dict[str, Any], + trace: Trace +) -> Feedback: + """Check if the correct tools were called.""" + + expected_tools = expectations.get("expected_tools", []) + + if not expected_tools: + return Feedback( + name="tool_selection_accuracy", + value="skip", + rationale="No expected_tools in expectations" + ) + + # Get actual tool calls from TOOL spans + tool_spans = trace.search_spans(span_type=SpanType.TOOL) + actual_tools = {span.name for span in tool_spans} + + # Normalize names (handle fully qualified names like "catalog.schema.func") + def normalize(name: str) -> str: + return name.split(".")[-1] if "." in name else name + + expected_normalized = {normalize(t) for t in expected_tools} + actual_normalized = {normalize(t) for t in actual_tools} + + # Check if all expected tools were called + missing = expected_normalized - actual_normalized + + all_expected_called = len(missing) == 0 + + rationale = f"Expected: {list(expected_normalized)}, Actual: {list(actual_normalized)}" + if missing: + rationale += f" | Missing: {list(missing)}" + + return Feedback( + name="tool_selection_accuracy", + value="yes" if all_expected_called else "no", + rationale=rationale + ) +``` + +### Pattern: Stage Latency Scorer (Multiple Metrics) + +Measure latency per pipeline stage and identify bottlenecks. + +```python +from mlflow.genai.scorers import scorer +from mlflow.entities import Feedback, Trace +from typing import List + +@scorer +def stage_latency_scorer(trace: Trace) -> List[Feedback]: + """Measure latency for each pipeline stage.""" + + feedbacks = [] + all_spans = trace.search_spans() + + # Total trace time + root_spans = [s for s in all_spans if s.parent_id is None] + if root_spans: + root = root_spans[0] + total_ms = (root.end_time_ns - root.start_time_ns) / 1e6 + feedbacks.append(Feedback( + name="total_latency_ms", + value=round(total_ms, 2), + rationale=f"Total execution time: {total_ms:.2f}ms" + )) + + # Per-stage latency (customize patterns for your pipeline) + stage_patterns = ["classifier", "rewriter", "executor", "retriever"] + stage_times = {} + + for span in all_spans: + span_name_lower = span.name.lower() + for pattern in stage_patterns: + if pattern in span_name_lower: + duration_ms = (span.end_time_ns - span.start_time_ns) / 1e6 + stage_times[pattern] = stage_times.get(pattern, 0) + duration_ms + break + + for stage, time_ms in stage_times.items(): + feedbacks.append(Feedback( + name=f"{stage}_latency_ms", + value=round(time_ms, 2), + rationale=f"Stage '{stage}' took {time_ms:.2f}ms" + )) + + return feedbacks +``` + +### Pattern: Scorer with Aggregations + +Use for numeric scorers that need aggregate statistics. + +```python +from mlflow.genai.scorers import scorer + +@scorer(aggregations=["mean", "min", "max", "median", "p90"]) +def response_latency(outputs) -> float: + """Return response generation time.""" + return outputs.get("latency_ms", 0) / 1000.0 # Convert to seconds + +# Valid aggregations: min, max, mean, median, variance, p90 +# NOTE: p50, p99, sum are NOT valid - use median instead of p50 +``` + +### Pattern: Conditional Scoring Based on Input + +Apply different guidelines based on query type. + +```python +from mlflow.genai.scorers import scorer, Guidelines + +@scorer +def conditional_scorer(inputs, outputs): + """Apply different guidelines based on query type.""" + + query = inputs.get("query", "").lower() + + if "technical" in query or "how to" in query: + # Technical queries need detailed responses + judge = Guidelines( + name="technical_quality", + guidelines=[ + "Response must include step-by-step instructions", + "Response must include code examples where relevant" + ] + ) + elif "price" in query or "cost" in query: + # Pricing queries need specific info + judge = Guidelines( + name="pricing_quality", + guidelines=[ + "Response must include specific pricing information", + "Response must mention any conditions or limitations" + ] + ) + else: + # General queries + judge = Guidelines( + name="general_quality", + guidelines=[ + "Response must directly address the question", + "Response must be clear and concise" + ] + ) + + return judge(inputs=inputs, outputs=outputs) +``` + +--- + **For troubleshooting scorer issues**, see `references/troubleshooting.md` diff --git a/agent-evaluation/references/tracing-integration.md b/agent-evaluation/references/tracing-integration.md index 594fd35..7d99158 100644 --- a/agent-evaluation/references/tracing-integration.md +++ b/agent-evaluation/references/tracing-integration.md @@ -162,3 +162,165 @@ After completing all steps above, verify: - [ ] Test run creates traces with non-None trace_id (verified with validate_agent_tracing.py) - [ ] Traces visible in MLflow UI or via `mlflow traces search` - [ ] Trace hierarchy includes both @mlflow.trace spans and autolog spans + +--- + +## Trace Analysis Patterns + +Once tracing is integrated, use these patterns to analyze trace data for debugging and evaluation. + +### Search Traces + +```python +import mlflow +import time + +# All traces in current experiment +all_traces = mlflow.search_traces() + +# Successful traces only +ok_traces = mlflow.search_traces( + filter_string="attributes.status = 'OK'" +) + +# Error traces only +error_traces = mlflow.search_traces( + filter_string="attributes.status = 'ERROR'" +) + +# Recent traces (last hour) +one_hour_ago = int((time.time() - 3600) * 1000) +recent = mlflow.search_traces( + filter_string=f"attributes.timestamp_ms > {one_hour_ago}" +) + +# Slow traces (> 5 seconds) +slow = mlflow.search_traces( + filter_string="attributes.execution_time_ms > 5000" +) + +# By tag +prod_traces = mlflow.search_traces( + filter_string="tags.environment = 'production'" +) + +# By trace name (note backticks for dotted names) +specific_app = mlflow.search_traces( + filter_string="tags.`mlflow.traceName` = 'my_app_function'" +) +``` + +### Latency Breakdown by Span Type + +```python +from mlflow.entities import Trace, SpanType +from collections import defaultdict + +def latency_by_span_type(trace: Trace) -> dict: + """Break down latency by span type (LLM, TOOL, RETRIEVER, etc.)""" + spans = trace.search_spans() + type_latencies = defaultdict(list) + + for span in spans: + duration_ms = (span.end_time_ns - span.start_time_ns) / 1e6 + span_type = str(span.span_type) if span.span_type else "UNKNOWN" + type_latencies[span_type].append(duration_ms) + + results = {} + for span_type, durations in type_latencies.items(): + results[span_type] = { + "count": len(durations), + "total_ms": round(sum(durations), 2), + "avg_ms": round(sum(durations) / len(durations), 2), + } + return results +``` + +### Find Bottlenecks + +```python +def find_bottlenecks(trace: Trace, top_n: int = 5) -> list: + """Find the slowest spans in a trace.""" + spans = trace.search_spans() + exclude_patterns = ["forward", "predict", "root"] + + span_timings = [] + for span in spans: + span_name_lower = span.name.lower() + if any(p in span_name_lower for p in exclude_patterns): + continue + + duration_ms = (span.end_time_ns - span.start_time_ns) / 1e6 + span_timings.append({ + "name": span.name, + "span_type": str(span.span_type) if span.span_type else "UNKNOWN", + "duration_ms": round(duration_ms, 2), + }) + + span_timings.sort(key=lambda x: -x["duration_ms"]) + return span_timings[:top_n] +``` + +### Analyze Tool Calls + +```python +from mlflow.entities import SpanType + +def analyze_tool_calls(trace: Trace) -> dict: + """Analyze tool calls in a trace.""" + spans = trace.search_spans() + tool_spans = [s for s in spans if s.span_type == SpanType.TOOL] + + tool_calls = [] + for span in tool_spans: + duration_ms = (span.end_time_ns - span.start_time_ns) / 1e6 + tool_name = span.name.split(".")[-1] if "." in span.name else span.name + + tool_calls.append({ + "tool_name": tool_name, + "duration_ms": round(duration_ms, 2), + "inputs": span.inputs, + }) + + return { + "total_tool_calls": len(tool_calls), + "calls": tool_calls, + } +``` + +### Detect Errors + +```python +from mlflow.entities import SpanStatusCode + +def detect_errors(trace: Trace) -> dict: + """Detect error patterns in a trace.""" + spans = trace.search_spans() + + errors = { + "failed_spans": [], + "empty_outputs": [], + } + + for span in spans: + if span.status and span.status.status_code == SpanStatusCode.ERROR: + errors["failed_spans"].append({ + "name": span.name, + "error": span.status.description if span.status.description else "Unknown" + }) + + if span.outputs is None or span.outputs == {} or span.outputs == []: + errors["empty_outputs"].append({"name": span.name}) + + return errors +``` + +### Filter Syntax Reference + +| Syntax Element | Rule | +|----------------|------| +| String values | Use single quotes: `'OK'` NOT `"OK"` | +| Dotted names | Use backticks: `tags.\`mlflow.traceName\`` | +| Prefix | Required for attributes: `attributes.status` | +| Logical operators | `AND` supported, `OR` NOT supported | +| Time values | Use milliseconds since epoch | diff --git a/agent-evaluation/references/user-journeys.md b/agent-evaluation/references/user-journeys.md new file mode 100644 index 0000000..36ee989 --- /dev/null +++ b/agent-evaluation/references/user-journeys.md @@ -0,0 +1,285 @@ +# User Journey Guides + +Step-by-step workflows for common evaluation scenarios. + +--- + +## Journey 0: Strategy Alignment (ALWAYS START HERE) + +**Starting Point**: You need to evaluate an agent +**Goal**: Align on what to evaluate before writing any code + +**PRIORITY:** Before writing evaluation code, complete strategy alignment. This ensures evaluations measure what matters and provide actionable insights. + +### Step 1: Understand the Agent + +Before evaluating, gather context about what you're evaluating: + +**Questions to ask (or investigate in the codebase):** +1. **What does this agent do?** (data analysis, RAG, multi-turn chat, task automation) +2. **What tools does it use?** (UC functions, vector search, external APIs) +3. **What is the input/output format?** (messages format, structured output) +4. **What is the current state?** (prototype, production, needs improvement) + +**Actions to take:** +- Read the agent's main code file (e.g., `agent.py`) +- Review the config file for system prompts and tool definitions +- Check existing tests or evaluation scripts +- Look at CLAUDE.md or README for project context + +### Step 2: Align on What to Evaluate + +**Evaluation dimensions to consider:** + +| Dimension | When to Use | Example Scorer | +|-----------|-------------|----------------| +| **Safety** | Always (table stakes) | `Safety()` | +| **Correctness** | When ground truth exists | `Correctness()` | +| **Relevance** | When responses should address queries | `RelevanceToQuery()` | +| **Groundedness** | RAG systems with retrieved context | `RetrievalGroundedness()` | +| **Domain Guidelines** | Domain-specific requirements | `Guidelines(name="...", guidelines="...")` | +| **Format/Structure** | Structured output requirements | Custom scorer | +| **Tool Usage** | Agents with tool calls | Custom scorer checking tool selection | + +**Questions to ask the user:** +1. What are the **must-have** quality criteria? (safety, accuracy, relevance) +2. What are the **nice-to-have** criteria? (conciseness, tone, format) +3. Are there **specific failure modes** you've seen or worry about? +4. Do you have **ground truth** or expected answers for test cases? + +### Step 3: Define User Scenarios (Evaluation Dataset) + +**Types of test cases to include:** + +| Category | Purpose | Example | +|----------|---------|---------| +| **Happy Path** | Core functionality works | Typical user questions | +| **Edge Cases** | Boundary conditions | Empty inputs, very long queries | +| **Adversarial** | Robustness testing | Prompt injection, off-topic | +| **Multi-turn** | Conversation handling | Follow-up questions, context recall | +| **Domain-specific** | Business logic | Industry terminology, specific formats | + +**Questions to ask the user:** +1. What are the **most common** questions users ask? +2. What are **challenging** questions the agent should handle? +3. Are there questions it should **refuse** to answer? +4. Do you have **existing test cases** or production traces to start from? + +### Step 4: Establish Success Criteria + +**Define quality gates before running evaluation:** + +```python +QUALITY_GATES = { + "safety": 1.0, # 100% - non-negotiable + "correctness": 0.9, # 90% - high bar for accuracy + "relevance": 0.85, # 85% - good relevance + "concise": 0.8, # 80% - nice to have +} +``` + +**Questions to ask the user:** +1. What pass rates are **acceptable** for each dimension? +2. Which metrics are **blocking** vs **informational**? +3. How will evaluation results **inform decisions**? (ship/no-ship, iterate, investigate) + +### Strategy Alignment Checklist + +Before implementing evaluation, confirm: +- [ ] Agent purpose and architecture understood +- [ ] Evaluation dimensions agreed upon +- [ ] Test case categories identified +- [ ] Success criteria defined +- [ ] Data source identified (new, traces, existing dataset) + +--- + +## Journey 3: "Something Broke" - Regression Detection + +**Starting Point**: You made changes to your agent and suspect something regressed +**Goal**: Identify what broke and verify the fix + +### Steps + +1. **Establish baseline metrics** + ```bash + # Run evaluation on the previous version (or use saved baseline) + uv run python scripts/run_evaluation_template.py + ``` + Record key metrics from previous run + +2. **Run evaluation on current version** + ```bash + uv run python scripts/run_evaluation_template.py + ``` + +3. **Compare metrics** + - Load results from both runs + - Compare pass rates for each scorer + +4. **Identify regression source** + - If `classifier_accuracy` dropped -> Check ClassifierSignature changes + - If `tool_selection_accuracy` dropped -> Check tool descriptions, required_tools field + - If `follows_instructions` dropped -> Check ExecutorSignature output format + +5. **Analyze failing traces** + - Review traces that failed in current but passed in baseline + - Look for: + - Error patterns in specific test categories + - Tool call failures + - Unexpected outputs + +6. **Fix and re-evaluate** + - Revert problematic changes or apply targeted fix + - Re-run evaluation + - Verify metrics restored + +### Success Indicators +- Metrics return to baseline or improve +- No new failing test cases +- Trace analysis shows expected behavior + +--- + +## Journey 7: "My Multi-Agent is Slow" - Performance Optimization + +**Starting Point**: Your agent responses are too slow +**Goal**: Identify bottlenecks and reduce latency + +### Steps + +1. **Run evaluation with latency scoring** + ```bash + uv run python scripts/run_evaluation_template.py + ``` + Note the latency metrics: + - `classifier_latency_ms` + - `rewriter_latency_ms` + - `executor_latency_ms` + - `total_latency_ms` + +2. **Identify the bottleneck stage** + | Latency | Typical Range | If High, Check | + |---------|---------------|----------------| + | classifier_latency | <5s | ClassifierSignature verbosity | + | rewriter_latency | <10s | QueryRewriterSignature complexity | + | executor_latency | <30s | Tool call count, response generation | + +3. **Analyze traces for slow stages** + Focus on: + - Span durations by stage + - Number of LLM calls per stage + - Tool execution times + +4. **Run signature analysis** + Look for: + - High total description chars (>2000) + - Verbose OutputField descriptions + - Missing examples (causes more retries) + +5. **Apply optimizations** + + **For high classifier latency:** + - Simplify ClassifierSignature docstring + - Add concrete examples to reduce ambiguity + + **For high executor latency:** + - Simplify ExecutorSignature.answer format + - Reduce output format requirements + - Consider caching repeated tool calls + + **For high total latency:** + - Review if all stages are necessary + - Consider parallel execution where possible + +6. **Re-evaluate and compare** + ```bash + uv run python scripts/run_evaluation_template.py + ``` + Compare to verify improvement + +### Success Indicators +- Target latencies: classifier <5s, executor <30s, total <60s +- No regression in accuracy metrics +- Consistent improvement across test categories + +--- + +## Journey 8: "Improve My Prompts" - Systematic Prompt Optimization + +**Starting Point**: Your agent works but could be more accurate +**Goal**: Systematically improve prompt quality through evaluation + +### Steps + +1. **Establish baseline** + ```bash + uv run python scripts/run_evaluation_template.py + ``` + Record all metrics + +2. **Identify lowest-performing metrics** + | Metric | Primary Signature | Common Issues | + |--------|-------------------|---------------| + | follows_instructions | ExecutorSignature | Verbose answer format, unclear structure | + | tool_selection_accuracy | ClassifierSignature | No examples, ambiguous tool descriptions | + | classifier_accuracy | ClassifierSignature | Verbose docstring, unclear query_type mapping | + +3. **Apply ONE fix at a time** + - Make a single, targeted change + - Document the change in your commit message + - Track results + +4. **Re-evaluate immediately** + ```bash + uv run python scripts/run_evaluation_template.py + ``` + - If improved -> Keep change, move to next fix + - If regressed -> Revert and try different approach + - If unchanged -> Consider if fix was necessary + +5. **Iterate until targets met** + + | Metric | Target | + |--------|--------| + | classifier_accuracy | 95%+ | + | tool_selection_accuracy | 90%+ | + | follows_instructions | 80%+ | + +6. **Document successful optimizations** + Keep notes on what worked and why + +### Success Indicators +- All target metrics met +- No regressions from baseline +- Clear documentation of what changed and why + +--- + +## Quick Reference + +### Which Journey Am I On? + +| Symptom | Journey | +|---------|---------| +| "It was working before" | Journey 3 (Regression) | +| "It's too slow" | Journey 7 (Performance) | +| "It's not accurate enough" | Journey 8 (Prompt Optimization) | + +### Common Tools Across Journeys + +| Tool | Purpose | +|------|---------| +| `run_evaluation_template.py` | Run evaluation | +| `analyze_results.py` | Result analysis | +| `list_datasets.py` | Dataset discovery | +| `validate_environment.py` | Environment check | + +### Metric Targets + +| Metric | Target | Critical Threshold | +|--------|--------|-------------------| +| classifier_accuracy | 95%+ | <80% | +| tool_selection_accuracy | 90%+ | <70% | +| follows_instructions | 80%+ | <50% | +| executor_latency | <30s | >60s |