diff --git a/scenarios/agent-tracing/README.md b/scenarios/agent-tracing/README.md new file mode 100644 index 00000000..15ac22c0 --- /dev/null +++ b/scenarios/agent-tracing/README.md @@ -0,0 +1,31 @@ +# Agent Tracing + +Reasoning about agent executions is critical for troubleshooting and debugging. Complex agents can involve many nested steps, variable execution paths, and long inputs/outputs, which makes it hard to pinpoint issues. Tracing provides a clear, chronological view of the inputs and outputs for each primitive involved in a run. + +This scenario sets up a structure for agent tracing using OpenTelemetry. It supports: +- Local tracing via console or any OTLP-compatible backend (e.g., Aspire Dashboard). +- Cloud tracing via Azure Monitor when Application Insights is enabled for your Azure AI Studio project. + +## Structure +- `langchain/`: Tracing patterns for LangChain flows. +- `langgraph/`: Tracing patterns for LangGraph agents/graphs. +- `openai-agents/`: Tracing for OpenAI Agents with Azure OpenAI. + +Each subfolder contains its own `requirements.txt`, optional `dev-requirements.txt`, and `.env.sample` tailored for that sample. + +## Prerequisites +- Python 3.10+ recommended. +- An Azure AI Studio project (optional, for Azure Monitor tracing). +- If using Azure Monitor, enable the Tracing tab in your AI Studio project to provision Application Insights and retrieve the connection string. + +## Installation +Navigate to a subfolder and install its `requirements.txt`. Use `dev-requirements.txt` if you want Azure Monitor integrations. + +## Configuration +- Local OTLP exporter: + - Set `OTEL_EXPORTER_OTLP_ENDPOINT` to your backend (e.g., Aspire Dashboard default: `http://localhost:4317` for gRPC, or `http://localhost:4318` for HTTP). +- Azure Monitor: + - Copy the subfolder `.env.sample` to `.env` and set required values. + +## Notes +- The initial release of Azure AI Projects had a known issue where agent function tool call details might be included in traces even when content recording is disabled. Be cautious with sensitive data and review Tracing settings in AI Studio. diff --git a/scenarios/agent-tracing/langchain/.env.sample b/scenarios/agent-tracing/langchain/.env.sample new file mode 100644 index 00000000..bf05b82f --- /dev/null +++ b/scenarios/agent-tracing/langchain/.env.sample @@ -0,0 +1,9 @@ +# Azure OpenAI configuration (required) +AZURE_OPENAI_API_KEY= +AZURE_OPENAI_ENDPOINT= +AZURE_OPENAI_DEPLOYMENT= +AZURE_OPENAI_API_VERSION=2024-02-15-preview + +# Optional: Enable Azure Monitor tracing via Application Insights +APPLICATION_INSIGHTS_CONNECTION_STRING= + diff --git a/scenarios/agent-tracing/langchain/README.md b/scenarios/agent-tracing/langchain/README.md new file mode 100644 index 00000000..9a526bd9 --- /dev/null +++ b/scenarios/agent-tracing/langchain/README.md @@ -0,0 +1,62 @@ +--- +page_type: sample +languages: +- python +products: +- ai-services +- azure-openai +description: Pure LangChain weather assistant with Azure tracing and manual tool-calling loop. +--- + +## Pure LangChain Weather (Tracing) + +### Overview + +This sample demonstrates a pure LangChain agent that uses a manual tool-calling loop, instrumented with Azure Application Insights via `langchain-azure-ai`. It calls Azure OpenAI chat models and a simple `get_weather` tool, and emits OpenTelemetry traces locally or to Azure Monitor. + +### Objective + +- Use `langchain` and `langchain-openai` with Azure OpenAI chat models. +- Add tracing via `langchain-azure-ai` and OpenTelemetry. +- Implement a manual tool-calling loop for clarity and control. + +### Programming Languages + +- Python + +### Estimated Runtime: 10 mins + +## Set up + +Create and activate a local virtual environment, then install dependencies: + +``` +python -m venv .venv +source .venv/bin/activate # Windows: .venv\\Scripts\\activate +pip install -r requirements.txt +``` + +Copy the environment template and set required variables: + +``` +cp .env.sample .env +``` + +Required: + +- `AZURE_OPENAI_API_KEY` +- `AZURE_OPENAI_ENDPOINT` +- `AZURE_OPENAI_DEPLOYMENT` +- `AZURE_OPENAI_API_VERSION` (default `2024-02-15-preview`) + +Optional for tracing: + +- `APPLICATION_INSIGHTS_CONNECTION_STRING` + +## Run + +``` +python weather.py +``` + +You can send traces to an OTLP-compatible backend by setting `OTEL_EXPORTER_OTLP_ENDPOINT`, or to Azure Monitor by setting `APPLICATION_INSIGHTS_CONNECTION_STRING`. diff --git a/scenarios/agent-tracing/langchain/dev-requirements.txt b/scenarios/agent-tracing/langchain/dev-requirements.txt new file mode 100644 index 00000000..3016f177 --- /dev/null +++ b/scenarios/agent-tracing/langchain/dev-requirements.txt @@ -0,0 +1,9 @@ +# Full dev setup (optional) +langchain +langchain-openai +langchain-azure-ai +opentelemetry-api +opentelemetry-sdk +opentelemetry-exporter-otlp +python-dotenv +azure-monitor-opentelemetry diff --git a/scenarios/agent-tracing/langchain/requirements.txt b/scenarios/agent-tracing/langchain/requirements.txt new file mode 100644 index 00000000..9fb42537 --- /dev/null +++ b/scenarios/agent-tracing/langchain/requirements.txt @@ -0,0 +1,7 @@ +langchain +langchain-openai +langchain-azure-ai +opentelemetry-api +opentelemetry-sdk +opentelemetry-exporter-otlp +python-dotenv diff --git a/scenarios/agent-tracing/langchain/weather.py b/scenarios/agent-tracing/langchain/weather.py new file mode 100644 index 00000000..f2bc0926 --- /dev/null +++ b/scenarios/agent-tracing/langchain/weather.py @@ -0,0 +1,215 @@ +""" +LangChain Weather Assistant with manual tool-calling loop + Azure tracing. + +Env vars required: + AZURE_OPENAI_API_KEY=... + AZURE_OPENAI_ENDPOINT=https://YOUR-RESOURCE.openai.azure.com + AZURE_OPENAI_DEPLOYMENT=yourDeploymentName + AZURE_OPENAI_API_VERSION=2024-02-15-preview (or compatible) + +Optional tracing: + APPLICATION_INSIGHTS_CONNECTION_STRING=InstrumentationKey=...;IngestionEndpoint=... + +Run: + python weather.py +""" + +import os +import json +import logging +from datetime import datetime +from typing import List, Any, Optional, Dict + +from langchain_core.tools import tool +from langchain_core.messages import ( + SystemMessage, + HumanMessage, + AIMessage, + ToolMessage, + BaseMessage, +) +from langchain_openai import AzureChatOpenAI + +try: + from langchain_azure_ai.callbacks.tracers import AzureAIOpenTelemetryTracer +except ImportError: + AzureAIOpenTelemetryTracer = None + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger("langchain_weather") + + +# ----------------------------------------------------------------------------- +# Tracing Setup (cached) +# ----------------------------------------------------------------------------- +_TRACERS: Optional[List[Any]] = None + + +def setup_tracing() -> List[Any]: + global _TRACERS + if _TRACERS is not None: + return _TRACERS + tracers: List[Any] = [] + conn = os.getenv("APPLICATION_INSIGHTS_CONNECTION_STRING") + if conn and AzureAIOpenTelemetryTracer: + try: + tracer = AzureAIOpenTelemetryTracer( + connection_string=conn, + enable_content_recording=True, + name="langchain_weather", + id="weather_agent", + ) + tracers.append(tracer) + logger.info("Azure tracing enabled.") + except Exception as e: + logger.warning(f"Failed to init tracer: {e}") + else: + logger.info("Tracing not enabled (missing APPLICATION_INSIGHTS_CONNECTION_STRING or dependency).") + _TRACERS = tracers + return tracers + + +def trace_config(agent_name: str, session_id: str) -> Dict[str, Any]: + tracers = setup_tracing() + return { + "callbacks": tracers, + "tags": [f"agent:{agent_name}", agent_name, "weather-langchain"], + "metadata": { + "agent_name": agent_name, + "agent_type": agent_name, + "langgraph_node": agent_name, # kept for parity + "session_id": session_id, + "thread_id": session_id, + "system": "langchain-weather", + }, + } + + +# ----------------------------------------------------------------------------- +# Tool +# ----------------------------------------------------------------------------- +@tool +def get_weather(location: str, date: Optional[str] = None) -> str: + """ + Return a mock weather forecast as JSON. + """ + if not date: + date = datetime.utcnow().strftime("%Y-%m-%d") + seed = sum(ord(c) for c in location.lower()) % 5 + conditions = ["Sunny", "Partly Cloudy", "Light Rain", "Overcast", "Showers"] + cond = conditions[seed] + forecast = { + "location": location, + "date": date, + "condition": cond, + "temp_high_c": 24 + seed, + "temp_low_c": 14 + seed, + "advice": "Great day outside!" if cond == "Sunny" else "Plan for changing conditions.", + } + return json.dumps(forecast, indent=2) + + +TOOLS = [get_weather] +TOOLS_BY_NAME = {t.name: t for t in TOOLS} + + +# ----------------------------------------------------------------------------- +# LLM Factory +# ----------------------------------------------------------------------------- +def build_llm(session_id: str) -> AzureChatOpenAI: + required = [ + "AZURE_OPENAI_API_KEY", + "AZURE_OPENAI_ENDPOINT", + "AZURE_OPENAI_DEPLOYMENT", + ] + missing = [v for v in required if not os.getenv(v)] + if missing: + raise RuntimeError(f"Missing Azure OpenAI env vars: {', '.join(missing)}") + return AzureChatOpenAI( + api_key=os.environ["AZURE_OPENAI_API_KEY"], + azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"], + azure_deployment=os.environ["AZURE_OPENAI_DEPLOYMENT"], + api_version=os.getenv("AZURE_OPENAI_API_VERSION", "2024-02-15-preview"), + temperature=0.2, + callbacks=setup_tracing(), + tags=["weather_agent", "weather-langchain"], + metadata={ + "agent_type": "weather_agent", + "agent_name": "weather_agent", + "system": "langchain-weather", + "session_id": session_id, + "thread_id": session_id, + }, + ) + + +SYSTEM_PROMPT = """You are a weather assistant. +If user asks about weather, call the get_weather tool with (location, date if given). +If ambiguous date, assume tomorrow. +After tool output, summarize succinctly for the user. +""" + + +# ----------------------------------------------------------------------------- +# Agent Loop (manual) +# ----------------------------------------------------------------------------- +def run_weather_conversation(user_query: str, session_id: str) -> str: + llm = build_llm(session_id) + # Bind tools for tool-calling (function-calling) capability + tool_llm = llm.bind_tools(TOOLS) + + messages: List[BaseMessage] = [ + SystemMessage(content=SYSTEM_PROMPT), + HumanMessage(content=user_query), + ] + + # We allow up to N reasoning/tool steps (simple guard) + for step in range(5): + logger.info(f"LLM step {step + 1}") + response: AIMessage = tool_llm.invoke(messages, config=trace_config("weather_agent", session_id)) + messages.append(response) + + # If the model decided not to call any tools, we stop + tool_calls = getattr(response, "tool_calls", None) + if not tool_calls: + logger.info("No tool calls; finishing.") + break + + # Execute each tool call and append ToolMessage + for tc in tool_calls: + name = tc["name"] + args = tc.get("args", {}) + tool_obj = TOOLS_BY_NAME.get(name) + if not tool_obj: + tool_output = f"Tool '{name}' not found." + else: + try: + tool_output = tool_obj.invoke(args) + except Exception as e: + tool_output = f"Error executing tool '{name}': {e}" + messages.append( + ToolMessage( + content=tool_output, + name=name, + tool_call_id=tc["id"], + ) + ) + + # Final answer: last AI message with no tool calls OR last AI message overall + final_ai = next((m for m in reversed(messages) if isinstance(m, AIMessage)), None) + return final_ai.content if final_ai else "No AI response." + + +def main() -> None: + print("Pure LangChain Weather (Instrumented)") + q = input("Ask a weather question (e.g. 'Weather in Tokyo tomorrow'): ").strip() + if not q: + q = "Weather in Paris" + session_id = f"lc-session-{datetime.utcnow().strftime('%Y%m%d%H%M%S')}" + answer = run_weather_conversation(q, session_id) + print("\n--- Answer ---") + print(answer) + + +if __name__ == "__main__": + main() diff --git a/scenarios/agent-tracing/langgraph/.env.sample b/scenarios/agent-tracing/langgraph/.env.sample new file mode 100644 index 00000000..bf05b82f --- /dev/null +++ b/scenarios/agent-tracing/langgraph/.env.sample @@ -0,0 +1,9 @@ +# Azure OpenAI configuration (required) +AZURE_OPENAI_API_KEY= +AZURE_OPENAI_ENDPOINT= +AZURE_OPENAI_DEPLOYMENT= +AZURE_OPENAI_API_VERSION=2024-02-15-preview + +# Optional: Enable Azure Monitor tracing via Application Insights +APPLICATION_INSIGHTS_CONNECTION_STRING= + diff --git a/scenarios/agent-tracing/langgraph/README.md b/scenarios/agent-tracing/langgraph/README.md new file mode 100644 index 00000000..c0617b2c --- /dev/null +++ b/scenarios/agent-tracing/langgraph/README.md @@ -0,0 +1,62 @@ +--- +page_type: sample +languages: +- python +products: +- ai-services +- azure-openai +description: Pure LangGraph weather workflow with Azure tracing and single-node tool execution. +--- + +## Pure LangGraph Weather (Tracing) + +### Overview + +This sample demonstrates a LangGraph single-node workflow that calls Azure OpenAI chat models and a `get_weather` tool, with traces exported via `langchain-azure-ai` to local OTLP endpoints or Azure Monitor. + +### Objective + +- Use `langgraph` with `langchain` and Azure OpenAI. +- Instrument with `langchain-azure-ai` and OpenTelemetry for tracing. +- Stream steps and inspect final state in a simple weather flow. + +### Programming Languages + +- Python + +### Estimated Runtime: 10 mins + +## Set up + +Create and activate a local virtual environment, then install dependencies: + +``` +python -m venv .venv +source .venv/bin/activate # Windows: .venv\\Scripts\\activate +pip install -r requirements.txt +``` + +Copy the environment template and set required variables: + +``` +cp .env.sample .env +``` + +Required: + +- `AZURE_OPENAI_API_KEY` +- `AZURE_OPENAI_ENDPOINT` +- `AZURE_OPENAI_DEPLOYMENT` +- `AZURE_OPENAI_API_VERSION` (optional; defaults to `2024-02-15-preview`) + +Optional for tracing: + +- `APPLICATION_INSIGHTS_CONNECTION_STRING` + +## Run + +``` +python weather.py +``` + +Optionally set `OTEL_EXPORTER_OTLP_ENDPOINT` for local OTLP backends, or `APPLICATION_INSIGHTS_CONNECTION_STRING` to send traces to Azure Monitor. diff --git a/scenarios/agent-tracing/langgraph/dev-requirements.txt b/scenarios/agent-tracing/langgraph/dev-requirements.txt new file mode 100644 index 00000000..65e8c41b --- /dev/null +++ b/scenarios/agent-tracing/langgraph/dev-requirements.txt @@ -0,0 +1,10 @@ +# Full dev setup (optional) +langchain +langchain-openai +langchain-azure-ai +langgraph +opentelemetry-api +opentelemetry-sdk +opentelemetry-exporter-otlp +python-dotenv +azure-monitor-opentelemetry diff --git a/scenarios/agent-tracing/langgraph/requirements.txt b/scenarios/agent-tracing/langgraph/requirements.txt new file mode 100644 index 00000000..1317833c --- /dev/null +++ b/scenarios/agent-tracing/langgraph/requirements.txt @@ -0,0 +1,8 @@ +langchain +langchain-openai +langchain-azure-ai +langgraph +opentelemetry-api +opentelemetry-sdk +opentelemetry-exporter-otlp +python-dotenv diff --git a/scenarios/agent-tracing/langgraph/weather.py b/scenarios/agent-tracing/langgraph/weather.py new file mode 100644 index 00000000..956f8042 --- /dev/null +++ b/scenarios/agent-tracing/langgraph/weather.py @@ -0,0 +1,259 @@ +""" +LangGraph single-node weather workflow with Azure tracing. + +Env vars required (same as langchain/weather.py): + AZURE_OPENAI_API_KEY + AZURE_OPENAI_ENDPOINT + AZURE_OPENAI_DEPLOYMENT +Optional: + AZURE_OPENAI_API_VERSION + APPLICATION_INSIGHTS_CONNECTION_STRING + +Run: + python weather.py +""" + +import os +import json +import logging +from datetime import datetime +from typing import TypedDict, List, Annotated, Optional, Any, Dict + +from langchain_core.tools import tool +from langchain_core.messages import ( + HumanMessage, + SystemMessage, + AIMessage, + ToolMessage, + AnyMessage, +) +from langgraph.graph import StateGraph, START, END +from langgraph.graph.message import add_messages +from langgraph.checkpoint.memory import MemorySaver +from langchain_openai import AzureChatOpenAI + +try: + from langchain_azure_ai.callbacks.tracers import AzureAIOpenTelemetryTracer +except ImportError: + AzureAIOpenTelemetryTracer = None + +logging.basicConfig(level=logging.INFO) +log = logging.getLogger("langgraph_weather") + + +# ----------------------------------------------------------------------------- +# Tracing +# ----------------------------------------------------------------------------- +_TRACERS: Optional[list[Any]] = None + + +def setup_tracing() -> list[Any]: + global _TRACERS + if _TRACERS is not None: + return _TRACERS + tracers: list[Any] = [] + conn = os.getenv("APPLICATION_INSIGHTS_CONNECTION_STRING") + if conn and AzureAIOpenTelemetryTracer: + try: + tracers.append( + AzureAIOpenTelemetryTracer( + connection_string=conn, + enable_content_recording=True, + name="langgraph_weather", + id="weather_graph_agent", + ) + ) + log.info("Azure tracing enabled.") + except Exception as e: + log.warning(f"Tracing init failed: {e}") + else: + log.info("Tracing disabled (no APPLICATION_INSIGHTS_CONNECTION_STRING).") + _TRACERS = tracers + return tracers + + +def trace_config(agent_name: str, session_id: str) -> Dict[str, Any]: + tracers = setup_tracing() + return { + "callbacks": tracers, + "tags": [f"agent:{agent_name}", agent_name, "weather-langgraph"], + "metadata": { + "agent_name": agent_name, + "agent_type": agent_name, + "langgraph_node": agent_name, + "session_id": session_id, + "thread_id": session_id, + "system": "langgraph-weather", + }, + } + + +# ----------------------------------------------------------------------------- +# Tool +# ----------------------------------------------------------------------------- +@tool +def get_weather(location: str, date: Optional[str] = None) -> str: + if not date: + date = datetime.utcnow().strftime("%Y-%m-%d") + seed = sum(ord(c) for c in location.lower()) % 4 + conds = ["Sunny", "Windy", "Showers", "Cloudy"] + cond = conds[seed] + return json.dumps( + { + "location": location, + "date": date, + "condition": cond, + "high_c": 23 + seed, + "low_c": 13 + seed, + "advice": "Bring a jacket." if cond != "Sunny" else "Enjoy the sunshine!", + }, + indent=2, + ) + + +TOOLS = [get_weather] +TOOLS_BY_NAME = {t.name: t for t in TOOLS} + + +# ----------------------------------------------------------------------------- +# State +# ----------------------------------------------------------------------------- +class WeatherState(TypedDict): + messages: Annotated[List[AnyMessage], add_messages] + session_id: str + done: bool + + +SYSTEM_PROMPT = """You are a weather assistant. +Use get_weather tool exactly once if user asks about conditions. +Return a concise summary referencing the tool output. +""" + + +# ----------------------------------------------------------------------------- +# LLM +# ----------------------------------------------------------------------------- +def build_llm(session_id: str) -> AzureChatOpenAI: + required = [ + "AZURE_OPENAI_API_KEY", + "AZURE_OPENAI_ENDPOINT", + "AZURE_OPENAI_DEPLOYMENT", + ] + missing = [v for v in required if not os.getenv(v)] + if missing: + raise RuntimeError(f"Missing Azure OpenAI env vars: {', '.join(missing)}") + return AzureChatOpenAI( + api_key=os.environ["AZURE_OPENAI_API_KEY"], + azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"], + azure_deployment=os.environ["AZURE_OPENAI_DEPLOYMENT"], + api_version=os.getenv("AZURE_OPENAI_API_VERSION", "2024-02-15-preview"), + temperature=0.2, + callbacks=setup_tracing(), + tags=["weather_agent", "weather-langgraph"], + metadata={ + "agent_type": "weather_agent", + "agent_name": "weather_agent", + "system": "langgraph-weather", + "session_id": session_id, + "thread_id": session_id, + }, + ) + + +# ----------------------------------------------------------------------------- +# Node +# ----------------------------------------------------------------------------- +def weather_node(state: WeatherState) -> WeatherState: + # If already done, just pass state (idempotency) + if state.get("done"): + return state + + llm = build_llm(state["session_id"]) + tool_llm = llm.bind_tools(TOOLS) + + # Step 1: ask the model + response: AIMessage = tool_llm.invoke( + state["messages"], + config=trace_config("weather_agent", state["session_id"]), + ) + state["messages"].append(response) + + tool_calls = getattr(response, "tool_calls", None) + if tool_calls: + # Execute tool calls + for tc in tool_calls: + name = tc["name"] + args = tc.get("args", {}) + tool_obj = TOOLS_BY_NAME.get(name) + if not tool_obj: + output = f"Tool '{name}' not found." + else: + try: + output = tool_obj.invoke(args) + except Exception as e: + output = f"Error executing tool '{name}': {e}" + state["messages"].append(ToolMessage(name=name, tool_call_id=tc["id"], content=output)) + # After tool outputs, ask model again to summarize + final_response: AIMessage = llm.invoke( + state["messages"], + config=trace_config("weather_agent", state["session_id"]), + ) + state["messages"].append(final_response) + + # Mark done + state["done"] = True + return state + + +# ----------------------------------------------------------------------------- +# Control Flow +# ----------------------------------------------------------------------------- +def route(state: WeatherState) -> str: + if not state.get("done"): + return "weather_agent" + return END + + +def build_app() -> StateGraph: + g = StateGraph(WeatherState) + g.add_node("weather_agent", weather_node) + g.add_conditional_edges(START, lambda _state: "weather_agent") + g.add_conditional_edges("weather_agent", route) + checkpointer = MemorySaver() + return g.compile(checkpointer=checkpointer) + + +# ----------------------------------------------------------------------------- +# CLI +# ----------------------------------------------------------------------------- +def main() -> None: + print("Pure LangGraph Weather (Instrumented)") + query = input("Ask a weather question: ").strip() + if not query: + query = "What's the weather in Lisbon tomorrow?" + session_id = f"graph-session-{datetime.utcnow().strftime('%Y%m%d%H%M%S')}" + + app = build_app() + initial: WeatherState = { + "messages": [ + SystemMessage(content=SYSTEM_PROMPT), + HumanMessage(content=query), + ], + "session_id": session_id, + "done": False, + } + + print("\n--- Streaming Steps ---") + for step in app.stream(initial, {"configurable": {"thread_id": session_id}}): + print(step) + + final_state = app.get_state({"configurable": {"thread_id": session_id}}) + all_msgs = final_state.values["messages"] + final_ai = [m for m in all_msgs if isinstance(m, AIMessage)] + if final_ai: + print("\n--- Final Answer ---") + print(final_ai[-1].content) + + +if __name__ == "__main__": + main() diff --git a/scenarios/agent-tracing/openai-agents/.env.sample b/scenarios/agent-tracing/openai-agents/.env.sample new file mode 100644 index 00000000..20bd37e2 --- /dev/null +++ b/scenarios/agent-tracing/openai-agents/.env.sample @@ -0,0 +1,8 @@ +# Azure OpenAI Agents (required) +AZURE_OPENAI_ENDPOINT= +AZURE_OPENAI_VERSION= +AZURE_OPENAI_CHAT_DEPLOYMENT= + +# Optional tracing +APPLICATION_INSIGHTS_CONNECTION_STRING= + diff --git a/scenarios/agent-tracing/openai-agents/README.md b/scenarios/agent-tracing/openai-agents/README.md new file mode 100644 index 00000000..54d31519 --- /dev/null +++ b/scenarios/agent-tracing/openai-agents/README.md @@ -0,0 +1,61 @@ +--- +page_type: sample +languages: +- python +products: +- ai-services +- azure-openai +description: OpenAI Agents sample (Spanish Tutor) instrumented with OpenTelemetry for console or Azure Monitor. +--- + +## OpenAI Agents Spanish Tutor (Tracing) + +### Overview + +This sample uses the `openai-agents` Python SDK with Azure OpenAI (Chat Completions) and instruments agent runs via OpenTelemetry. It authenticates using `DefaultAzureCredential` (supports `az login` or service principal) and exports spans either to Azure Monitor or the console. + +### Objective + +- Configure Azure OpenAI with the `openai` SDK and `openai-agents`. +- Instrument the Agents framework with `opentelemetry-instrumentation-openai-agents`. +- Export traces to Azure Monitor or console. + +### Programming Languages + +- Python + +### Estimated Runtime: 10 mins + +## Set up + +Create a virtual environment and install dependencies: + +``` +python -m venv .venv +source .venv/bin/activate # Windows: .venv\\Scripts\\activate +pip install -r requirements.txt +``` + +Copy the environment template and set required variables: + +``` +cp .env.sample .env +``` + +Required: + +- `AZURE_OPENAI_ENDPOINT` +- `AZURE_OPENAI_VERSION` +- `AZURE_OPENAI_CHAT_DEPLOYMENT` + +Optional tracing: + +- `APPLICATION_INSIGHTS_CONNECTION_STRING` + +## Run + +``` +API_HOST=azure python spanish_tutor.py +``` + +Provide `APPLICATION_INSIGHTS_CONNECTION_STRING` to export traces to Azure Monitor, otherwise spans are printed to console. Ensure you have `az login` or a valid service principal configured for `DefaultAzureCredential`. diff --git a/scenarios/agent-tracing/openai-agents/dev-requirements.txt b/scenarios/agent-tracing/openai-agents/dev-requirements.txt new file mode 100644 index 00000000..6370a177 --- /dev/null +++ b/scenarios/agent-tracing/openai-agents/dev-requirements.txt @@ -0,0 +1,9 @@ +# Full dev setup (optional) +openai>=1.35.0 +openai-agents>=0.2.9 +azure-identity>=1.17.0 +opentelemetry-api>=1.27.0 +opentelemetry-sdk>=1.27.0 +opentelemetry-instrumentation-openai-agents[instruments] @ git+https://github.com/nagkumar91/opentelemetry-python-contrib@main#subdirectory=instrumentation-genai/opentelemetry-instrumentation-openai-agents +python-dotenv>=1.0.1 +azure-monitor-opentelemetry-exporter diff --git a/scenarios/agent-tracing/openai-agents/requirements.txt b/scenarios/agent-tracing/openai-agents/requirements.txt new file mode 100644 index 00000000..9ba04af7 --- /dev/null +++ b/scenarios/agent-tracing/openai-agents/requirements.txt @@ -0,0 +1,8 @@ +openai>=1.35.0 +openai-agents>=0.2.9 +azure-identity>=1.17.0 +opentelemetry-api>=1.27.0 +opentelemetry-sdk>=1.27.0 +opentelemetry-instrumentation-openai-agents[instruments] @ git+https://github.com/nagkumar91/opentelemetry-python-contrib@main#subdirectory=instrumentation-genai/opentelemetry-instrumentation-openai-agents +python-dotenv>=1.0.1 +azure-monitor-opentelemetry-exporter diff --git a/scenarios/agent-tracing/openai-agents/spanish_tutor.py b/scenarios/agent-tracing/openai-agents/spanish_tutor.py new file mode 100644 index 00000000..fdb7ebd0 --- /dev/null +++ b/scenarios/agent-tracing/openai-agents/spanish_tutor.py @@ -0,0 +1,204 @@ +""" +Spanish Tutor (Azure OpenAI) + OpenTelemetry (Console or Azure Monitor) + +This script: + * Uses Azure OpenAI (Chat Completions) via the `openai` Python SDK (>=1.x) + * Authenticates with DefaultAzureCredential (so you can use az login or a service principal) + * Instruments the OpenAI Agents framework with OpenTelemetry + * Captures rich GenAI semantic attributes (messages, system instructions, tool definitions) + * Exports spans either to: + - Azure Monitor (if APPLICATION_INSIGHTS_CONNECTION_STRING is set), or + - Console (fallback) + +Prerequisites: + pip install: + openai + openai-agents + azure-identity + opentelemetry-sdk + opentelemetry-api + opentelemetry-instrumentation-openai-agents + (optional) azure-monitor-opentelemetry-exporter + +Run: + API_HOST=azure python spanish_tutor.py +""" + +from __future__ import annotations + +import asyncio +import logging +import os +from dataclasses import dataclass +from typing import Callable +from urllib.parse import urlparse + +import azure.identity +import openai +from agents import Agent, OpenAIChatCompletionsModel, Runner # from openai-agents + +from opentelemetry import trace +from opentelemetry.instrumentation.openai_agents import OpenAIAgentsInstrumentor +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import ( + BatchSpanProcessor, + ConsoleSpanExporter, +) + +logging.basicConfig(level=logging.INFO) + + +@dataclass +class _ApiConfig: + build_client: Callable[[], openai.AsyncAzureOpenAI] + model_name: str + base_url: str + provider: str + + +def _set_capture_env(_provider: str, base_url: str) -> None: + """ + Set OpenTelemetry + GenAI semantic capture environment toggles if not already provided. + These default to 'true' to showcase the richest possible trace payload. + """ + capture_defaults = { + # Enable instrumentation features + "OTEL_INSTRUMENTATION_OPENAI_AGENTS_CAPTURE_CONTENT": "true", + "OTEL_INSTRUMENTATION_OPENAI_AGENTS_CAPTURE_METRICS": "true", + # Generic GenAI semantic attrs capture + "OTEL_GENAI_CAPTURE_MESSAGES": "true", + "OTEL_GENAI_CAPTURE_SYSTEM_INSTRUCTIONS": "true", + "OTEL_GENAI_CAPTURE_TOOL_DEFINITIONS": "true", + "OTEL_GENAI_EMIT_OPERATION_DETAILS": "true", + # Agent identity metadata + "OTEL_GENAI_AGENT_NAME": os.getenv("OTEL_GENAI_AGENT_NAME", "Spanish Tutor Agent"), + "OTEL_GENAI_AGENT_DESCRIPTION": os.getenv( + "OTEL_GENAI_AGENT_DESCRIPTION", + "Conversational tutor that always replies in Spanish", + ), + "OTEL_GENAI_AGENT_ID": os.getenv("OTEL_GENAI_AGENT_ID", "spanish-tutor"), + } + for k, v in capture_defaults.items(): + os.environ.setdefault(k, v) + + # 'provider' is used for identity metadata in env; parsing base_url sets server attrs + parsed = urlparse(base_url) + if parsed.hostname: + os.environ.setdefault("OTEL_GENAI_SERVER_ADDRESS", parsed.hostname) + if parsed.port: + os.environ.setdefault("OTEL_GENAI_SERVER_PORT", str(parsed.port)) + + +def _resolve_api_config() -> _ApiConfig: + """ + For this Azure-focused sample we expect API_HOST=azure. + """ + host = os.getenv("API_HOST", "azure").lower() + if host != "azure": + raise ValueError("This sample is locked to API_HOST=azure for clarity.") + + endpoint = os.environ["AZURE_OPENAI_ENDPOINT"].rstrip("/") + api_version = os.environ["AZURE_OPENAI_VERSION"] + deployment = os.environ["AZURE_OPENAI_CHAT_DEPLOYMENT"] + + credential = azure.identity.DefaultAzureCredential() + token_provider = azure.identity.get_bearer_token_provider( + credential, + "https://cognitiveservices.azure.com/.default", + ) + + def _build_client() -> openai.AsyncAzureOpenAI: + return openai.AsyncAzureOpenAI( + api_version=api_version, + azure_endpoint=endpoint, + azure_ad_token_provider=token_provider, + ) + + return _ApiConfig( + build_client=_build_client, + model_name=deployment, + base_url=endpoint, + provider="azure.ai.openai", + ) + + +def _configure_otel() -> None: + """ + Configure TracerProvider + exporter. + If APPLICATION_INSIGHTS_CONNECTION_STRING is set, export to Azure Monitor. + Otherwise, export spans to the console. + """ + conn = os.getenv("APPLICATION_INSIGHTS_CONNECTION_STRING") + resource = Resource.create( + { + "service.name": os.getenv("OTEL_SERVICE_NAME", "spanish-tutor-app"), + "service.namespace": "language-learning", + "service.version": os.getenv("SERVICE_VERSION", "1.0.0"), + } + ) + + tracer_provider = TracerProvider(resource=resource) + + if conn: + try: + from azure.monitor.opentelemetry.exporter import ( # type: ignore + AzureMonitorTraceExporter, + ) + except ImportError: + print( + "Azure Monitor exporter not installed. Falling back to console. " + "Install with: pip install azure-monitor-opentelemetry-exporter" + ) + tracer_provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter())) + else: + tracer_provider.add_span_processor( + BatchSpanProcessor(AzureMonitorTraceExporter.from_connection_string(conn)) + ) + print("[otel] Azure Monitor trace exporter configured") + else: + tracer_provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter())) + print("[otel] Console span exporter configured") + print("[otel] Provide APPLICATION_INSIGHTS_CONNECTION_STRING to export to Azure Monitor.") + + trace.set_tracer_provider(tracer_provider) + + +def _infer_span_name(provider: str) -> str: + return f"spanish_tutor_session[{provider}]" + + +async def main() -> None: + api_config = _resolve_api_config() + _set_capture_env(api_config.provider, api_config.base_url) + _configure_otel() + + # Instrument AFTER setting tracer provider + OpenAIAgentsInstrumentor().instrument(tracer_provider=trace.get_tracer_provider()) + + client = api_config.build_client() + + agent = Agent( + name="Spanish tutor", + instructions="You are a Spanish tutor. Help the user learn Spanish. ONLY respond in Spanish.", + model=OpenAIChatCompletionsModel( + model=api_config.model_name, + openai_client=client, + ), + ) + + tracer = trace.get_tracer(__name__) + # Create a session span + with tracer.start_as_current_span(_infer_span_name(api_config.provider)): + result = await Runner.run(agent, input="Hola, ¿cómo estás?") + print("\n=== Final Tutor Reply (Spanish) ===") + print(result.final_output) + print("==================================\n") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + finally: + # Ensure processors flush + trace.get_tracer_provider().shutdown() diff --git a/scenarios/evaluate/Simulators/Simulate_Evaluate_Groundedness/Simulate_Evaluate_Groundedness.ipynb b/scenarios/evaluate/Simulators/Simulate_Evaluate_Groundedness/Simulate_Evaluate_Groundedness.ipynb index ccb85314..0c6b11e2 100644 --- a/scenarios/evaluate/Simulators/Simulate_Evaluate_Groundedness/Simulate_Evaluate_Groundedness.ipynb +++ b/scenarios/evaluate/Simulators/Simulate_Evaluate_Groundedness/Simulate_Evaluate_Groundedness.ipynb @@ -85,9 +85,9 @@ "outputs": [], "source": [ "azure_ai_project = {\n", - " \"subscription_id\": \"\",\n", - " \"resource_group\": \"\",\n", - " \"workspace_name\": \"\",\n", + " \"subscription_id\": \"AZURE_SUBSCRIPTION_ID\",\n", + " \"resource_group_name\": \"RESOURCE_GROUP\",\n", + " \"project_name\": \"PROJECT_NAME\",\n", "}\n", "\n", "azure_openai_endpoint = \"\"\n", diff --git a/scenarios/langchain/tracing-with-langchain.ipynb b/scenarios/langchain/tracing-with-langchain.ipynb index 98163df8..752f3492 100644 --- a/scenarios/langchain/tracing-with-langchain.ipynb +++ b/scenarios/langchain/tracing-with-langchain.ipynb @@ -135,9 +135,9 @@ "metadata": {}, "outputs": [], "source": [ - "from langchain_azure_ai.callbacks.tracers import AzureAIInferenceTracer\n", + "from langchain_azure_ai.callbacks.tracers import AzureAIOpenTelemetryTracer\n", "\n", - "tracer = AzureAIInferenceTracer(\n", + "tracer = AzureAIOpenTelemetryTracer(\n", " connection_string=application_insights_connection_string,\n", " enable_content_recording=True,\n", ")"